From 6f2c1047c500a7395035ce9a6e373b15b3379856 Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Wed, 4 Jan 2023 18:23:52 +0800 Subject: [PATCH 01/38] add yaml for fedex in PubMed --- .../usability/pubmed/bo_gp.yaml | 0 .../usability/pubmed/bo_kde.yaml | 0 .../usability/pubmed/bo_rf.yaml | 0 .../usability/pubmed/bohb.yaml | 0 .../hpo_exp_scripts/usability/pubmed/hb.yaml | 0 .../hpo_exp_scripts/usability/pubmed/rs.yaml | 53 ++++++++++++++++ .../usability/pubmed/rs_wrap.yaml | 60 +++++++++++++++++++ .../hpo_exp_scripts/usability/pubmed/ss.yaml | 16 +++++ 8 files changed, 129 insertions(+) create mode 100644 scripts/hpo_exp_scripts/usability/pubmed/bo_gp.yaml create mode 100644 scripts/hpo_exp_scripts/usability/pubmed/bo_kde.yaml create mode 100644 scripts/hpo_exp_scripts/usability/pubmed/bo_rf.yaml create mode 100644 scripts/hpo_exp_scripts/usability/pubmed/bohb.yaml create mode 100644 scripts/hpo_exp_scripts/usability/pubmed/hb.yaml create mode 100644 scripts/hpo_exp_scripts/usability/pubmed/rs.yaml create mode 100644 scripts/hpo_exp_scripts/usability/pubmed/rs_wrap.yaml create mode 100644 scripts/hpo_exp_scripts/usability/pubmed/ss.yaml diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bo_gp.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bo_gp.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bo_kde.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bo_kde.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bo_rf.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bo_rf.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bohb.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bohb.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/hpo_exp_scripts/usability/pubmed/hb.yaml b/scripts/hpo_exp_scripts/usability/pubmed/hb.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/hpo_exp_scripts/usability/pubmed/rs.yaml b/scripts/hpo_exp_scripts/usability/pubmed/rs.yaml new file mode 100644 index 000000000..4aa6032c1 --- /dev/null +++ b/scripts/hpo_exp_scripts/usability/pubmed/rs.yaml @@ -0,0 +1,53 @@ +use_gpu: True +device: 2 +outdir: rs_pubmed_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + make_global_eval: True + client_num: 5 + total_round_num: 500 + share_local_model: True + online_aggr: True +data: + root: data/ + type: pubmed + splitter: 'louvain' +model: + type: gcn + hidden: 64 + dropout: 0.5 + out_channels: 3 + task: node +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +criterion: + type: CrossEntropyLoss +trainer: + type: nodefullbatch_trainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +dataloader: + type: pyg + batch_size: 1 +hpo: + scheduler: rs + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + table: + num: 27 + metric: 'server_global_eval.val_avg_loss' + working_folder: rs_pubmed_avg \ No newline at end of file diff --git a/scripts/hpo_exp_scripts/usability/pubmed/rs_wrap.yaml b/scripts/hpo_exp_scripts/usability/pubmed/rs_wrap.yaml new file mode 100644 index 000000000..75b8c89db --- /dev/null +++ b/scripts/hpo_exp_scripts/usability/pubmed/rs_wrap.yaml @@ -0,0 +1,60 @@ +use_gpu: True +device: 2 +outdir: rs_wrap_pubmed_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + make_global_eval: True + client_num: 5 + total_round_num: 500 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: pubmed + splitter: 'louvain' +model: + type: gcn + hidden: 64 + dropout: 0.5 + out_channels: 3 + task: node +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +criterion: + type: CrossEntropyLoss +trainer: + type: nodefullbatch_trainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +dataloader: + type: pyg + batch_size: 1 +hpo: + scheduler: wrap_rs + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + table: + num: 27 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'server_global_eval.val_avg_loss' + working_folder: rs_wrap_pubmed_avg diff --git a/scripts/hpo_exp_scripts/usability/pubmed/ss.yaml b/scripts/hpo_exp_scripts/usability/pubmed/ss.yaml new file mode 100644 index 000000000..af60a261a --- /dev/null +++ b/scripts/hpo_exp_scripts/usability/pubmed/ss.yaml @@ -0,0 +1,16 @@ +train.optimizer.lr: + type: float + lower: 0.01 + upper: 1.0 + log: True +train.optimizer.weight_decay: + type: float + lower: 0.0 + upper: 1.0 +model.dropout: + type: cate + choices: [0.0, 0.5] +train.local_update_steps: + type: int + lower: 1 + upper: 8 From 894c7c38ba55106558c072f4f56178869bbbdf31 Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Fri, 6 Jan 2023 11:19:58 +0800 Subject: [PATCH 02/38] init commit for hpo scripts --- .../usability/pubmed/bo_gp.yaml | 53 ++++++++++++++++ .../usability/pubmed/bo_gp_wrap.yaml | 60 +++++++++++++++++++ .../usability/pubmed/bo_kde.yaml | 53 ++++++++++++++++ .../usability/pubmed/bo_kde_wrap.yaml | 60 +++++++++++++++++++ .../usability/pubmed/bo_rf.yaml | 53 ++++++++++++++++ .../usability/pubmed/bo_rf_wrap.yaml | 60 +++++++++++++++++++ .../usability/pubmed/bohb.yaml | 53 ++++++++++++++++ .../usability/pubmed/bohb_wrap.yaml | 60 +++++++++++++++++++ .../hpo_exp_scripts/usability/pubmed/hb.yaml | 53 ++++++++++++++++ .../usability/pubmed/hb_wrap.yaml | 60 +++++++++++++++++++ 10 files changed, 565 insertions(+) create mode 100644 scripts/hpo_exp_scripts/usability/pubmed/bo_gp_wrap.yaml create mode 100644 scripts/hpo_exp_scripts/usability/pubmed/bo_kde_wrap.yaml create mode 100644 scripts/hpo_exp_scripts/usability/pubmed/bo_rf_wrap.yaml create mode 100644 scripts/hpo_exp_scripts/usability/pubmed/bohb_wrap.yaml create mode 100644 scripts/hpo_exp_scripts/usability/pubmed/hb_wrap.yaml diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bo_gp.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bo_gp.yaml index e69de29bb..44501956f 100644 --- a/scripts/hpo_exp_scripts/usability/pubmed/bo_gp.yaml +++ b/scripts/hpo_exp_scripts/usability/pubmed/bo_gp.yaml @@ -0,0 +1,53 @@ +use_gpu: True +device: 2 +outdir: bo_gp_pubmed_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + make_global_eval: True + client_num: 5 + total_round_num: 500 + share_local_model: True + online_aggr: True +data: + root: data/ + type: pubmed + splitter: 'louvain' +model: + type: gcn + hidden: 64 + dropout: 0.5 + out_channels: 3 + task: node +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +criterion: + type: CrossEntropyLoss +trainer: + type: nodefullbatch_trainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +dataloader: + type: pyg + batch_size: 1 +hpo: + scheduler: bo_gp + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + table: + num: 27 + metric: 'server_global_eval.val_avg_loss' + working_folder: bo_gp_pubmed_avg \ No newline at end of file diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bo_gp_wrap.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bo_gp_wrap.yaml new file mode 100644 index 000000000..61467c046 --- /dev/null +++ b/scripts/hpo_exp_scripts/usability/pubmed/bo_gp_wrap.yaml @@ -0,0 +1,60 @@ +use_gpu: True +device: 2 +outdir: bo_gp_wrap_pubmed_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + make_global_eval: True + client_num: 5 + total_round_num: 500 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: pubmed + splitter: 'louvain' +model: + type: gcn + hidden: 64 + dropout: 0.5 + out_channels: 3 + task: node +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +criterion: + type: CrossEntropyLoss +trainer: + type: nodefullbatch_trainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +dataloader: + type: pyg + batch_size: 1 +hpo: + scheduler: wrap_bo_gp + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + table: + num: 27 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'server_global_eval.val_avg_loss' + working_folder: bo_gp_wrap_pubmed_avg diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bo_kde.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bo_kde.yaml index e69de29bb..fef15d9a3 100644 --- a/scripts/hpo_exp_scripts/usability/pubmed/bo_kde.yaml +++ b/scripts/hpo_exp_scripts/usability/pubmed/bo_kde.yaml @@ -0,0 +1,53 @@ +use_gpu: True +device: 2 +outdir: bo_kde_pubmed_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + make_global_eval: True + client_num: 5 + total_round_num: 500 + share_local_model: True + online_aggr: True +data: + root: data/ + type: pubmed + splitter: 'louvain' +model: + type: gcn + hidden: 64 + dropout: 0.5 + out_channels: 3 + task: node +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +criterion: + type: CrossEntropyLoss +trainer: + type: nodefullbatch_trainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +dataloader: + type: pyg + batch_size: 1 +hpo: + scheduler: bo_kde + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + table: + num: 27 + metric: 'server_global_eval.val_avg_loss' + working_folder: bo_kde_pubmed_avg \ No newline at end of file diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bo_kde_wrap.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bo_kde_wrap.yaml new file mode 100644 index 000000000..de3a45ad5 --- /dev/null +++ b/scripts/hpo_exp_scripts/usability/pubmed/bo_kde_wrap.yaml @@ -0,0 +1,60 @@ +use_gpu: True +device: 2 +outdir: bo_kde_wrap_pubmed_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + make_global_eval: True + client_num: 5 + total_round_num: 500 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: pubmed + splitter: 'louvain' +model: + type: gcn + hidden: 64 + dropout: 0.5 + out_channels: 3 + task: node +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +criterion: + type: CrossEntropyLoss +trainer: + type: nodefullbatch_trainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +dataloader: + type: pyg + batch_size: 1 +hpo: + scheduler: wrap_bo_kde + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + table: + num: 27 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'server_global_eval.val_avg_loss' + working_folder: bo_kde_wrap_pubmed_avg diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bo_rf.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bo_rf.yaml index e69de29bb..c07591a71 100644 --- a/scripts/hpo_exp_scripts/usability/pubmed/bo_rf.yaml +++ b/scripts/hpo_exp_scripts/usability/pubmed/bo_rf.yaml @@ -0,0 +1,53 @@ +use_gpu: True +device: 2 +outdir: bo_rf_pubmed_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + make_global_eval: True + client_num: 5 + total_round_num: 500 + share_local_model: True + online_aggr: True +data: + root: data/ + type: pubmed + splitter: 'louvain' +model: + type: gcn + hidden: 64 + dropout: 0.5 + out_channels: 3 + task: node +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +criterion: + type: CrossEntropyLoss +trainer: + type: nodefullbatch_trainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +dataloader: + type: pyg + batch_size: 1 +hpo: + scheduler: bo_rf + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + table: + num: 27 + metric: 'server_global_eval.val_avg_loss' + working_folder: bo_rf_pubmed_avg \ No newline at end of file diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bo_rf_wrap.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bo_rf_wrap.yaml new file mode 100644 index 000000000..bd130b773 --- /dev/null +++ b/scripts/hpo_exp_scripts/usability/pubmed/bo_rf_wrap.yaml @@ -0,0 +1,60 @@ +use_gpu: True +device: 2 +outdir: bo_rf_wrap_pubmed_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + make_global_eval: True + client_num: 5 + total_round_num: 500 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: pubmed + splitter: 'louvain' +model: + type: gcn + hidden: 64 + dropout: 0.5 + out_channels: 3 + task: node +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +criterion: + type: CrossEntropyLoss +trainer: + type: nodefullbatch_trainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +dataloader: + type: pyg + batch_size: 1 +hpo: + scheduler: wrap_bo_rf + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + table: + num: 27 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'server_global_eval.val_avg_loss' + working_folder: bo_rf_wrap_pubmed_avg diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bohb.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bohb.yaml index e69de29bb..858ebc55a 100644 --- a/scripts/hpo_exp_scripts/usability/pubmed/bohb.yaml +++ b/scripts/hpo_exp_scripts/usability/pubmed/bohb.yaml @@ -0,0 +1,53 @@ +use_gpu: True +device: 2 +outdir: bohb_pubmed_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + make_global_eval: True + client_num: 5 + total_round_num: 500 + share_local_model: True + online_aggr: True +data: + root: data/ + type: pubmed + splitter: 'louvain' +model: + type: gcn + hidden: 64 + dropout: 0.5 + out_channels: 3 + task: node +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +criterion: + type: CrossEntropyLoss +trainer: + type: nodefullbatch_trainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +dataloader: + type: pyg + batch_size: 1 +hpo: + scheduler: bohb + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' + sha: + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 + table: + num: 27 + metric: 'server_global_eval.val_avg_loss' + working_folder: bohb_pubmed_avg \ No newline at end of file diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bohb_wrap.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bohb_wrap.yaml new file mode 100644 index 000000000..2fde86467 --- /dev/null +++ b/scripts/hpo_exp_scripts/usability/pubmed/bohb_wrap.yaml @@ -0,0 +1,60 @@ +use_gpu: True +device: 2 +outdir: bohb_wrap_pubmed_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + make_global_eval: True + client_num: 5 + total_round_num: 500 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: pubmed + splitter: 'louvain' +model: + type: gcn + hidden: 64 + dropout: 0.5 + out_channels: 3 + task: node +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +criterion: + type: CrossEntropyLoss +trainer: + type: nodefullbatch_trainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +dataloader: + type: pyg + batch_size: 1 +hpo: + scheduler: wrap_bohb + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' + sha: + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 + table: + num: 27 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'server_global_eval.val_avg_loss' + working_folder: bohb_wrap_pubmed_avg diff --git a/scripts/hpo_exp_scripts/usability/pubmed/hb.yaml b/scripts/hpo_exp_scripts/usability/pubmed/hb.yaml index e69de29bb..82b634bb0 100644 --- a/scripts/hpo_exp_scripts/usability/pubmed/hb.yaml +++ b/scripts/hpo_exp_scripts/usability/pubmed/hb.yaml @@ -0,0 +1,53 @@ +use_gpu: True +device: 2 +outdir: hb_pubmed_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + make_global_eval: True + client_num: 5 + total_round_num: 500 + share_local_model: True + online_aggr: True +data: + root: data/ + type: pubmed + splitter: 'louvain' +model: + type: gcn + hidden: 64 + dropout: 0.5 + out_channels: 3 + task: node +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +criterion: + type: CrossEntropyLoss +trainer: + type: nodefullbatch_trainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +dataloader: + type: pyg + batch_size: 1 +hpo: + scheduler: hb + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' + sha: + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 + table: + num: 27 + metric: 'server_global_eval.val_avg_loss' + working_folder: hb_pubmed_avg \ No newline at end of file diff --git a/scripts/hpo_exp_scripts/usability/pubmed/hb_wrap.yaml b/scripts/hpo_exp_scripts/usability/pubmed/hb_wrap.yaml new file mode 100644 index 000000000..d5d2e2ecf --- /dev/null +++ b/scripts/hpo_exp_scripts/usability/pubmed/hb_wrap.yaml @@ -0,0 +1,60 @@ +use_gpu: True +device: 2 +outdir: hb_wrap_pubmed_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + make_global_eval: True + client_num: 5 + total_round_num: 500 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: pubmed + splitter: 'louvain' +model: + type: gcn + hidden: 64 + dropout: 0.5 + out_channels: 3 + task: node +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +criterion: + type: CrossEntropyLoss +trainer: + type: nodefullbatch_trainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +dataloader: + type: pyg + batch_size: 1 +hpo: + scheduler: wrap_hb + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' + sha: + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 + table: + num: 27 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'server_global_eval.val_avg_loss' + working_folder: hb_wrap_pubmed_avg From eaf93ccb1e4791d0da86c88eccdda87e7f7c0a54 Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Fri, 6 Jan 2023 12:13:06 +0800 Subject: [PATCH 03/38] add run bash --- scripts/hpo_exp_scripts/usability/pubmed/run.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 scripts/hpo_exp_scripts/usability/pubmed/run.sh diff --git a/scripts/hpo_exp_scripts/usability/pubmed/run.sh b/scripts/hpo_exp_scripts/usability/pubmed/run.sh new file mode 100644 index 000000000..ea23f3c04 --- /dev/null +++ b/scripts/hpo_exp_scripts/usability/pubmed/run.sh @@ -0,0 +1,11 @@ +set -e + +seed=$1 + +methods=('rs' 'rs_wrap' 'bo_gp' 'bo_gp_wrap' 'bo_kde' 'bo_kde_wrap' 'bo_rf' 'bo_rf_wrap' 'hb' 'hb_wrap' 'bohb' 'bohb_wrap') + + +for (( m=0; m<${#methods[@]}; m++ )) +do + python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/usability/pubmed/${methods[$m]}.yaml device $((m%4)) seed ${seed}>${methods[$m]}_${seed}.log & +done From 8bbc209ed0cc412b60a33ba589faf8e2504fb2ea Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Fri, 6 Jan 2023 12:16:29 +0800 Subject: [PATCH 04/38] add nohup --- scripts/hpo_exp_scripts/usability/pubmed/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/hpo_exp_scripts/usability/pubmed/run.sh b/scripts/hpo_exp_scripts/usability/pubmed/run.sh index ea23f3c04..be5a8deb0 100644 --- a/scripts/hpo_exp_scripts/usability/pubmed/run.sh +++ b/scripts/hpo_exp_scripts/usability/pubmed/run.sh @@ -7,5 +7,5 @@ methods=('rs' 'rs_wrap' 'bo_gp' 'bo_gp_wrap' 'bo_kde' 'bo_kde_wrap' 'bo_rf' 'bo_ for (( m=0; m<${#methods[@]}; m++ )) do - python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/usability/pubmed/${methods[$m]}.yaml device $((m%4)) seed ${seed}>${methods[$m]}_${seed}.log & + nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/usability/pubmed/${methods[$m]}.yaml device $((m%4)) seed ${seed}>${methods[$m]}_${seed}.log & done From f6724e25a987e6f898d1e9e135d989cfcf10a139 Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Fri, 6 Jan 2023 17:37:11 +0800 Subject: [PATCH 05/38] add scripts for learning from scratch --- .../pubmed/learn_from_scratch/pubmed.yaml | 40 +++++++++++++++++ .../pubmed/learn_from_scratch/run.sh | 45 +++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml create mode 100644 scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/run.sh diff --git a/scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml b/scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml new file mode 100644 index 000000000..d20ef1ccd --- /dev/null +++ b/scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml @@ -0,0 +1,40 @@ +use_gpu: True +device: 2 +outdir: rs_pubmed_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + make_global_eval: True + client_num: 5 + total_round_num: 500 + share_local_model: True + online_aggr: True +data: + root: data/ + type: pubmed + splitter: 'louvain' +model: + type: gcn + hidden: 64 + dropout: 0.5 + out_channels: 3 + task: node +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +criterion: + type: CrossEntropyLoss +trainer: + type: nodefullbatch_trainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +dataloader: + type: pyg + batch_size: 1 diff --git a/scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/run.sh b/scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/run.sh new file mode 100644 index 000000000..3b9987637 --- /dev/null +++ b/scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/run.sh @@ -0,0 +1,45 @@ +set -e + +# Seed 12345 +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 7 train.optimizer.lr 0.224973 train.optimizer.weight_decay 0.001659 device 0 seed 12345>rs_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 1 train.optimizer.lr 0.7170358969751284 train.optimizer.weight_decay 0.0 device 1 seed 12345>rs_wrap_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 4 train.optimizer.lr 0.999311 train.optimizer.weight_decay 8.674074e-07 device 2 seed 12345>bo_gp_12345.log & +# >rs_wrap_12345.log >bo_gp_wrap_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 7 train.optimizer.lr 0.46701 train.optimizer.weight_decay 0.000853 device 0 seed 12345>bo_kde_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 7 train.optimizer.lr 0.46852259676975294 train.optimizer.weight_decay 0.263664919539593 device 1 seed 12345>bo_kde_wrap_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 6 train.optimizer.lr 0.984828 train.optimizer.weight_decay 0.00233 device 2 seed 12345>bo_rf_12345.log & +# >rs_wrap_12345.log >bo_rf_wrap_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 7 train.optimizer.lr 0.224973 train.optimizer.weight_decay 0.001659 device 0 seed 12345>hb_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 4 train.optimizer.lr 0.453625286588245 train.optimizer.weight_decay 0.06289557213350952 device 1 seed 12345>hb_wrap_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 7 train.optimizer.lr 0.224973 train.optimizer.weight_decay 0.001659 device 2 seed 12345>bohb_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 6 train.optimizer.lr 0.14495809658233344 train.optimizer.weight_decay 0.04477752881699114 device 3 seed 12345>bohb_wrap_12345.log & + +# Seed 12346 +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 8 train.optimizer.lr 0.75764 train.optimizer.weight_decay 0.007947 device 0 seed 12346>rs_12346.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 7 train.optimizer.lr 0.11480094171726124 train.optimizer.weight_decay 0.15500565663984123 device 1 seed 12346>rs_wrap_12346.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 3 train.optimizer.lr 0.999636 train.optimizer.weight_decay 9.7e-05 device 2 seed 12346>bo_gp_12346.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 2 train.optimizer.lr 0.6231045010965764 train.optimizer.weight_decay 0.020651830583413487 device 3 seed 12346>bo_gp_wrap_12346.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 7 train.optimizer.lr 0.094777 train.optimizer.weight_decay 0.019946 device 0 seed 12346>bo_kde_12346.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 7 train.optimizer.lr 0.09470439245585746 train.optimizer.weight_decay 0.1134997725167636 device 1 seed 12346>bo_kde_wrap_12346.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 3 train.optimizer.lr 0.991872 train.optimizer.weight_decay 0.000695 device 2 seed 12346>bo_rf_12346.log & +# >bo_gp_wrap_12346.log >bo_rf_wrap_12346.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 8 train.optimizer.lr 0.75764 train.optimizer.weight_decay 0.007947 device 0 seed 12346>hb_12346.log & +# >bo_kde_wrap_12346.log >hb_wrap_12346.log & +# >hb_12346.log >bohb_12346.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 7 train.optimizer.lr 0.19783377983946915 train.optimizer.weight_decay 0.3087280858375391 device 3 seed 12346>bohb_wrap_12346.log & + +# Seed 12347 +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 4 train.optimizer.lr 0.110196 train.optimizer.weight_decay 0.028109 device 0 seed 12347>rs_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 3 train.optimizer.lr 0.043324742759307006 train.optimizer.weight_decay 0.0 device 1 seed 12347>rs_wrap_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 8 train.optimizer.lr 0.999856 train.optimizer.weight_decay 1.2e-05 device 2 seed 12347>bo_gp_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 2 train.optimizer.lr 0.05512741305783294 train.optimizer.weight_decay 0.32323708666212847 device 3 seed 12347>bo_gp_wrap_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 2 train.optimizer.lr 0.020666 train.optimizer.weight_decay 0.010875 device 0 seed 12347>bo_kde_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 1 train.optimizer.lr 0.029033027691434364 train.optimizer.weight_decay 0.7362670096896027 device 1 seed 12347>bo_kde_wrap_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 8 train.optimizer.lr 0.998016 train.optimizer.weight_decay 0.001249 device 2 seed 12347>bo_rf_12347.log & +# >bo_gp_wrap_12347.log >bo_rf_wrap_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 4 train.optimizer.lr 0.110196 train.optimizer.weight_decay 0.028109 device 0 seed 12347>hb_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 5 train.optimizer.lr 0.07982026000525726 train.optimizer.weight_decay 0.07891495031558063 device 1 seed 12347>hb_wrap_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 7 train.optimizer.lr 0.015962 train.optimizer.weight_decay 1.5e-05 device 2 seed 12347>bohb_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 1 train.optimizer.lr 0.04927136943461437 train.optimizer.weight_decay 0.08747080914279645 device 3 seed 12347>bohb_wrap_12347.log & + + From 17bf4b1211a65aab0f92df617dcd47a79f0c2c87 Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Fri, 6 Jan 2023 17:41:49 +0800 Subject: [PATCH 06/38] add scripts for learning from scratch --- .../pubmed/learn_from_scratch/run.sh | 62 +++++++++---------- 1 file changed, 30 insertions(+), 32 deletions(-) diff --git a/scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/run.sh b/scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/run.sh index 3b9987637..cf0f2064c 100644 --- a/scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/run.sh +++ b/scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/run.sh @@ -1,45 +1,43 @@ set -e # Seed 12345 -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 7 train.optimizer.lr 0.224973 train.optimizer.weight_decay 0.001659 device 0 seed 12345>rs_12345.log & -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 1 train.optimizer.lr 0.7170358969751284 train.optimizer.weight_decay 0.0 device 1 seed 12345>rs_wrap_12345.log & -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 4 train.optimizer.lr 0.999311 train.optimizer.weight_decay 8.674074e-07 device 2 seed 12345>bo_gp_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 7 train.optimizer.lr 0.224973 train.optimizer.weight_decay 0.001659 device 0 seed 12345 >rs_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 1 train.optimizer.lr 0.7170358969751284 train.optimizer.weight_decay 0.0 device 1 seed 12345 >rs_wrap_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 4 train.optimizer.lr 0.999311 train.optimizer.weight_decay 8.674074e-07 device 2 seed 12345 >bo_gp_12345.log & # >rs_wrap_12345.log >bo_gp_wrap_12345.log & -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 7 train.optimizer.lr 0.46701 train.optimizer.weight_decay 0.000853 device 0 seed 12345>bo_kde_12345.log & -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 7 train.optimizer.lr 0.46852259676975294 train.optimizer.weight_decay 0.263664919539593 device 1 seed 12345>bo_kde_wrap_12345.log & -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 6 train.optimizer.lr 0.984828 train.optimizer.weight_decay 0.00233 device 2 seed 12345>bo_rf_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 7 train.optimizer.lr 0.46701 train.optimizer.weight_decay 0.000853 device 0 seed 12345 >bo_kde_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 7 train.optimizer.lr 0.46852259676975294 train.optimizer.weight_decay 0.263664919539593 device 1 seed 12345 >bo_kde_wrap_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 6 train.optimizer.lr 0.984828 train.optimizer.weight_decay 0.00233 device 2 seed 12345 >bo_rf_12345.log & # >rs_wrap_12345.log >bo_rf_wrap_12345.log & -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 7 train.optimizer.lr 0.224973 train.optimizer.weight_decay 0.001659 device 0 seed 12345>hb_12345.log & -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 4 train.optimizer.lr 0.453625286588245 train.optimizer.weight_decay 0.06289557213350952 device 1 seed 12345>hb_wrap_12345.log & -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 7 train.optimizer.lr 0.224973 train.optimizer.weight_decay 0.001659 device 2 seed 12345>bohb_12345.log & -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 6 train.optimizer.lr 0.14495809658233344 train.optimizer.weight_decay 0.04477752881699114 device 3 seed 12345>bohb_wrap_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 7 train.optimizer.lr 0.224973 train.optimizer.weight_decay 0.001659 device 0 seed 12345 >hb_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 4 train.optimizer.lr 0.453625286588245 train.optimizer.weight_decay 0.06289557213350952 device 1 seed 12345 >hb_wrap_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 7 train.optimizer.lr 0.224973 train.optimizer.weight_decay 0.001659 device 2 seed 12345 >bohb_12345.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 6 train.optimizer.lr 0.14495809658233344 train.optimizer.weight_decay 0.04477752881699114 device 3 seed 12345 >bohb_wrap_12345.log & # Seed 12346 -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 8 train.optimizer.lr 0.75764 train.optimizer.weight_decay 0.007947 device 0 seed 12346>rs_12346.log & -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 7 train.optimizer.lr 0.11480094171726124 train.optimizer.weight_decay 0.15500565663984123 device 1 seed 12346>rs_wrap_12346.log & -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 3 train.optimizer.lr 0.999636 train.optimizer.weight_decay 9.7e-05 device 2 seed 12346>bo_gp_12346.log & -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 2 train.optimizer.lr 0.6231045010965764 train.optimizer.weight_decay 0.020651830583413487 device 3 seed 12346>bo_gp_wrap_12346.log & -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 7 train.optimizer.lr 0.094777 train.optimizer.weight_decay 0.019946 device 0 seed 12346>bo_kde_12346.log & -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 7 train.optimizer.lr 0.09470439245585746 train.optimizer.weight_decay 0.1134997725167636 device 1 seed 12346>bo_kde_wrap_12346.log & -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 3 train.optimizer.lr 0.991872 train.optimizer.weight_decay 0.000695 device 2 seed 12346>bo_rf_12346.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 8 train.optimizer.lr 0.75764 train.optimizer.weight_decay 0.007947 device 0 seed 12346 >rs_12346.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 7 train.optimizer.lr 0.11480094171726124 train.optimizer.weight_decay 0.15500565663984123 device 1 seed 12346 >rs_wrap_12346.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 3 train.optimizer.lr 0.999636 train.optimizer.weight_decay 9.7e-05 device 2 seed 12346 >bo_gp_12346.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 2 train.optimizer.lr 0.6231045010965764 train.optimizer.weight_decay 0.020651830583413487 device 3 seed 12346 >bo_gp_wrap_12346.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 7 train.optimizer.lr 0.094777 train.optimizer.weight_decay 0.019946 device 0 seed 12346 >bo_kde_12346.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 7 train.optimizer.lr 0.09470439245585746 train.optimizer.weight_decay 0.1134997725167636 device 1 seed 12346 >bo_kde_wrap_12346.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 3 train.optimizer.lr 0.991872 train.optimizer.weight_decay 0.000695 device 2 seed 12346 >bo_rf_12346.log & # >bo_gp_wrap_12346.log >bo_rf_wrap_12346.log & -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 8 train.optimizer.lr 0.75764 train.optimizer.weight_decay 0.007947 device 0 seed 12346>hb_12346.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 8 train.optimizer.lr 0.75764 train.optimizer.weight_decay 0.007947 device 0 seed 12346 >hb_12346.log & # >bo_kde_wrap_12346.log >hb_wrap_12346.log & # >hb_12346.log >bohb_12346.log & -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 7 train.optimizer.lr 0.19783377983946915 train.optimizer.weight_decay 0.3087280858375391 device 3 seed 12346>bohb_wrap_12346.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 7 train.optimizer.lr 0.19783377983946915 train.optimizer.weight_decay 0.3087280858375391 device 3 seed 12346 >bohb_wrap_12346.log & # Seed 12347 -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 4 train.optimizer.lr 0.110196 train.optimizer.weight_decay 0.028109 device 0 seed 12347>rs_12347.log & -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 3 train.optimizer.lr 0.043324742759307006 train.optimizer.weight_decay 0.0 device 1 seed 12347>rs_wrap_12347.log & -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 8 train.optimizer.lr 0.999856 train.optimizer.weight_decay 1.2e-05 device 2 seed 12347>bo_gp_12347.log & -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 2 train.optimizer.lr 0.05512741305783294 train.optimizer.weight_decay 0.32323708666212847 device 3 seed 12347>bo_gp_wrap_12347.log & -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 2 train.optimizer.lr 0.020666 train.optimizer.weight_decay 0.010875 device 0 seed 12347>bo_kde_12347.log & -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 1 train.optimizer.lr 0.029033027691434364 train.optimizer.weight_decay 0.7362670096896027 device 1 seed 12347>bo_kde_wrap_12347.log & -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 8 train.optimizer.lr 0.998016 train.optimizer.weight_decay 0.001249 device 2 seed 12347>bo_rf_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 4 train.optimizer.lr 0.110196 train.optimizer.weight_decay 0.028109 device 0 seed 12347 >rs_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 3 train.optimizer.lr 0.043324742759307006 train.optimizer.weight_decay 0.0 device 1 seed 12347 >rs_wrap_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 8 train.optimizer.lr 0.999856 train.optimizer.weight_decay 1.2e-05 device 2 seed 12347 >bo_gp_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 2 train.optimizer.lr 0.05512741305783294 train.optimizer.weight_decay 0.32323708666212847 device 3 seed 12347 >bo_gp_wrap_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 2 train.optimizer.lr 0.020666 train.optimizer.weight_decay 0.010875 device 0 seed 12347 >bo_kde_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 1 train.optimizer.lr 0.029033027691434364 train.optimizer.weight_decay 0.7362670096896027 device 1 seed 12347 >bo_kde_wrap_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 8 train.optimizer.lr 0.998016 train.optimizer.weight_decay 0.001249 device 2 seed 12347 >bo_rf_12347.log & # >bo_gp_wrap_12347.log >bo_rf_wrap_12347.log & -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 4 train.optimizer.lr 0.110196 train.optimizer.weight_decay 0.028109 device 0 seed 12347>hb_12347.log & -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 5 train.optimizer.lr 0.07982026000525726 train.optimizer.weight_decay 0.07891495031558063 device 1 seed 12347>hb_wrap_12347.log & -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 7 train.optimizer.lr 0.015962 train.optimizer.weight_decay 1.5e-05 device 2 seed 12347>bohb_12347.log & -nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 1 train.optimizer.lr 0.04927136943461437 train.optimizer.weight_decay 0.08747080914279645 device 3 seed 12347>bohb_wrap_12347.log & - - +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 4 train.optimizer.lr 0.110196 train.optimizer.weight_decay 0.028109 device 0 seed 12347 >hb_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 5 train.optimizer.lr 0.07982026000525726 train.optimizer.weight_decay 0.07891495031558063 device 1 seed 12347 >hb_wrap_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.5 train.local_update_steps 7 train.optimizer.lr 0.015962 train.optimizer.weight_decay 1.5e-05 device 2 seed 12347 >bohb_12347.log & +nohup python federatedscope/main.py --cfg scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml model.dropout 0.0 train.local_update_steps 1 train.optimizer.lr 0.04927136943461437 train.optimizer.weight_decay 0.08747080914279645 device 3 seed 12347 >bohb_wrap_12347.log & From 256912c9cd6962e9e0a9a7912e1382549383d4a4 Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Mon, 9 Jan 2023 14:42:22 +0800 Subject: [PATCH 07/38] add scripts for run alpha --- .../uniqueness/cifar10/bo_gp.yaml | 56 ++++++++++++++++ .../uniqueness/cifar10/bo_gp_wrap.yaml | 66 +++++++++++++++++++ .../uniqueness/cifar10/bo_kde.yaml | 56 ++++++++++++++++ .../uniqueness/cifar10/bo_kde_wrap.yaml | 66 +++++++++++++++++++ .../uniqueness/cifar10/bo_rf.yaml | 56 ++++++++++++++++ .../uniqueness/cifar10/bo_rf_wrap.yaml | 66 +++++++++++++++++++ .../uniqueness/cifar10/bohb.yaml | 56 ++++++++++++++++ .../uniqueness/cifar10/bohb_wrap.yaml | 66 +++++++++++++++++++ .../uniqueness/cifar10/hb.yaml | 56 ++++++++++++++++ .../uniqueness/cifar10/hb_wrap.yaml | 66 +++++++++++++++++++ .../cifar10/learn_from_scratch/cifar10.yaml | 46 +++++++++++++ .../uniqueness/cifar10/rs.yaml | 57 ++++++++++++++++ .../uniqueness/cifar10/rs_wrap.yaml | 66 +++++++++++++++++++ .../hpo_exp_scripts/uniqueness/cifar10/run.sh | 12 ++++ .../uniqueness/cifar10/ss.yaml | 19 ++++++ 15 files changed, 810 insertions(+) create mode 100644 scripts/hpo_exp_scripts/uniqueness/cifar10/bo_gp.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/cifar10/bo_gp_wrap.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/cifar10/bo_kde.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/cifar10/bo_kde_wrap.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/cifar10/bo_rf.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/cifar10/bo_rf_wrap.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/cifar10/bohb.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/cifar10/bohb_wrap.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/cifar10/hb.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/cifar10/hb_wrap.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/cifar10/rs.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/cifar10/rs_wrap.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh create mode 100644 scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_gp.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_gp.yaml new file mode 100644 index 000000000..c703a1abb --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_gp.yaml @@ -0,0 +1,56 @@ +use_gpu: True +device: 3 +outdir: bo_gp_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: bo_gp + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_gp_cifar10_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_gp_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_gp_wrap.yaml new file mode 100644 index 000000000..b98e2f488 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_gp_wrap.yaml @@ -0,0 +1,66 @@ +use_gpu: True +device: 3 +outdir: bo_gp_wrap_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_bo_gp + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + table: + num: 27 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_gp_wrap_cifar10_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_kde.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_kde.yaml new file mode 100644 index 000000000..c21d7da72 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_kde.yaml @@ -0,0 +1,56 @@ +use_gpu: True +device: 3 +outdir: bo_kde_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: bo_kde + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_kde_cifar10_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_kde_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_kde_wrap.yaml new file mode 100644 index 000000000..588778c17 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_kde_wrap.yaml @@ -0,0 +1,66 @@ +use_gpu: True +device: 3 +outdir: bo_kde_wrap_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_bo_kde + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + table: + num: 27 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_kde_wrap_cifar10_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_rf.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_rf.yaml new file mode 100644 index 000000000..cf11c2178 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_rf.yaml @@ -0,0 +1,56 @@ +use_gpu: True +device: 3 +outdir: bo_rf_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: bo_rf + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_rf_cifar10_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_rf_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_rf_wrap.yaml new file mode 100644 index 000000000..f2b54e306 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_rf_wrap.yaml @@ -0,0 +1,66 @@ +use_gpu: True +device: 3 +outdir: bo_rf_wrap_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_bo_rf + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + table: + num: 27 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_rf_wrap_cifar10_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bohb.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bohb.yaml new file mode 100644 index 000000000..cd820c13e --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bohb.yaml @@ -0,0 +1,56 @@ +use_gpu: True +device: 3 +outdir: bohb_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: bohb + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bohb_cifar10_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bohb_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bohb_wrap.yaml new file mode 100644 index 000000000..e7bd45b97 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bohb_wrap.yaml @@ -0,0 +1,66 @@ +use_gpu: True +device: 3 +outdir: bohb_wrap_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_bohb + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 + table: + num: 27 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bohb_wrap_cifar10_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/hb.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/hb.yaml new file mode 100644 index 000000000..180e41bf5 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/hb.yaml @@ -0,0 +1,56 @@ +use_gpu: True +device: 3 +outdir: hb_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: hb + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: hb_cifar10_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/hb_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/hb_wrap.yaml new file mode 100644 index 000000000..11242ce85 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/hb_wrap.yaml @@ -0,0 +1,66 @@ +use_gpu: True +device: 3 +outdir: hb_wrap_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_hb + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 + table: + num: 27 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: hb_wrap_cifar10_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10.yaml new file mode 100644 index 000000000..21026a3e5 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10.yaml @@ -0,0 +1,46 @@ +use_gpu: True +device: 0 +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/rs.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/rs.yaml new file mode 100644 index 000000000..f4e073f4c --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/rs.yaml @@ -0,0 +1,57 @@ +use_gpu: True +device: 3 +outdir: rs_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: rs + num_workers: 0 + init_cand_num: 10 + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: rs_cifar10_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/rs_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/rs_wrap.yaml new file mode 100644 index 000000000..e9f9fcc81 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/rs_wrap.yaml @@ -0,0 +1,66 @@ +use_gpu: True +device: 3 +outdir: rs_wrap_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_rs + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + table: + num: 27 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: rs_wrap_cifar10_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh b/scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh new file mode 100644 index 000000000..2dba0ca66 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh @@ -0,0 +1,12 @@ +set -e + +alpha=$1 +seed=$2 + +methods=('rs' 'rs_wrap' 'bo_gp' 'bo_gp_wrap' 'bo_kde' 'bo_kde_wrap' 'bo_rf' 'bo_rf_wrap' 'hb' 'hb_wrap' 'bohb' 'bohb_wrap') + + +for (( m=0; m<${#methods[@]}; m++ )) +do + nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/uniqueness/cifar10/${methods[$m]}.yaml device $((m%4)) seed ${seed} data.splitter_args "[{'alpha': ${alpha}}]" >/dev/null 2>&1 & +done diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml new file mode 100644 index 000000000..e7d57b0ae --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml @@ -0,0 +1,19 @@ +train.optimizer.lr: + type: float + lower: 0.01 + upper: 1.0 + log: True +train.optimizer.weight_decay: + type: float + lower: 0.0 + upper: 1.0 +model.dropout: + type: cate + choices: [0.0, 0.5] +train.local_update_steps: + type: int + lower: 1 + upper: 4 +dataloader.batch_size: + type: cate + choices: [16, 32, 64] From ae3c388941e33cd8f2c9a290cdfc0b23488e048b Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Mon, 9 Jan 2023 15:41:18 +0800 Subject: [PATCH 08/38] fix minor bugs --- .../fedhpobench/optimizers/optuna_optimizer.py | 5 +++-- scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh | 13 +++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/benchmark/FedHPOBench/fedhpobench/optimizers/optuna_optimizer.py b/benchmark/FedHPOBench/fedhpobench/optimizers/optuna_optimizer.py index 0b0cb3cde..e81a50ac6 100644 --- a/benchmark/FedHPOBench/fedhpobench/optimizers/optuna_optimizer.py +++ b/benchmark/FedHPOBench/fedhpobench/optimizers/optuna_optimizer.py @@ -103,7 +103,6 @@ def objective(trial, benchmark, valid_budgets, configspace): cfg.benchmark.algo, device=cfg.benchmark.device) sampler = TPESampler(seed=cfg.optimizer.seed) - study = optuna.create_study(direction='minimize', sampler=sampler) if cfg.optimizer.type == 'tpe_md': pruner = MedianPruner() sh_iters = precompute_sh_iters(cfg.optimizer.min_budget, @@ -124,7 +123,9 @@ def objective(trial, benchmark, valid_budgets, configspace): ] else: raise NotImplementedError - + study = optuna.create_study(direction='minimize', + sampler=sampler, + pruner=pruner) study.optimize(func=partial( objective, benchmark=benchmark, diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh b/scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh index 2dba0ca66..c5faaa4bf 100644 --- a/scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh @@ -1,5 +1,18 @@ set -e +# How to use: +#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 0.05 12345 +#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 0.05 12346 +#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 0.05 12347 +# +#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 0.5 12345 +#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 0.5 12346 +#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 0.5 12347 +# +#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 5.0 12345 +#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 5.0 12346 +#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 5.0 12347 + alpha=$1 seed=$2 From c6a19db63442afe6a18771aee2e7882cf63e1f78 Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Mon, 9 Jan 2023 18:00:28 +0800 Subject: [PATCH 09/38] enable client_cfg --- federatedscope/autotune/algos.py | 32 +++++++++- federatedscope/autotune/utils.py | 24 ++++++- federatedscope/core/configs/cfg_hpo.py | 2 + federatedscope/hpo.py | 6 ++ .../uniqueness/personalized/bo_gp.yaml | 52 +++++++++++++++ .../uniqueness/personalized/bo_gp_wrap.yaml | 63 +++++++++++++++++++ .../uniqueness/personalized/bo_kde.yaml | 52 +++++++++++++++ .../uniqueness/personalized/bo_kde_wrap.yaml | 63 +++++++++++++++++++ .../uniqueness/personalized/bo_rf.yaml | 52 +++++++++++++++ .../uniqueness/personalized/bo_rf_wrap.yaml | 63 +++++++++++++++++++ .../uniqueness/personalized/bohb.yaml | 52 +++++++++++++++ .../uniqueness/personalized/bohb_wrap.yaml | 63 +++++++++++++++++++ .../uniqueness/personalized/hb.yaml | 52 +++++++++++++++ .../uniqueness/personalized/hb_wrap.yaml | 63 +++++++++++++++++++ .../uniqueness/personalized/rs.yaml | 53 ++++++++++++++++ .../uniqueness/personalized/rs_wrap.yaml | 63 +++++++++++++++++++ .../uniqueness/personalized/ss.yaml | 19 ++++++ 17 files changed, 770 insertions(+), 4 deletions(-) create mode 100644 scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp_wrap.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde_wrap.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf_wrap.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/personalized/bohb.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/personalized/bohb_wrap.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/personalized/hb.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/personalized/hb_wrap.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/personalized/rs.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/personalized/rs_wrap.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml diff --git a/federatedscope/autotune/algos.py b/federatedscope/autotune/algos.py index cf09c72ba..b2a05b6e8 100644 --- a/federatedscope/autotune/algos.py +++ b/federatedscope/autotune/algos.py @@ -13,8 +13,10 @@ from federatedscope.core.auxiliaries.worker_builder import get_client_cls, \ get_server_cls from federatedscope.core.auxiliaries.runner_builder import get_runner +from federatedscope.core.configs.yacs_config import CfgNode from federatedscope.autotune.utils import parse_search_space, \ - config2cmdargs, config2str, summarize_hpo_results, log2wandb + config2cmdargs, config2str, summarize_hpo_results, log2wandb, \ + flatten2nestdict logger = logging.getLogger(__name__) @@ -100,6 +102,16 @@ def __init__(self, cfg, client_cfgs=None): os.makedirs(self._cfg.hpo.working_folder, exist_ok=True) self._search_space = parse_search_space(self._cfg.hpo.ss) + # Convert to client_cfg + if self._cfg.hpo.personalized_ss: + # Do not support wrap_scheduler + client_num = self._cfg.federate.client_num + ss_client = CS.ConfigurationSpace() + for i in range(1, client_num + 1): + ss_client.add_configuration_space(f'client_{i}', + self._search_space, + delimiter='.') + self._search_space = ss_client self._init_configs = self._setup() logger.info(self._init_configs) @@ -159,7 +171,14 @@ def _evaluate(self, configs): thread_results[available_worker].clear() trial_cfg = self._cfg.clone() - trial_cfg.merge_from_list(config2cmdargs(config)) + if self._cfg.hpo.personalized_ss: + if isinstance(self._client_cfgs, CS.Configuration): + self._client_cfgs.merge_from_list( + config2cmdargs(config)) + else: + self._client_cfgs = CfgNode(flatten2nestdict(config)) + else: + trial_cfg.merge_from_list(config2cmdargs(config)) flags[available_worker].clear() trial = TrialExecutor(i, flags[available_worker], thread_results[available_worker], @@ -185,7 +204,14 @@ def _evaluate(self, configs): perfs = [None] * len(configs) for i, config in enumerate(configs): trial_cfg = self._cfg.clone() - trial_cfg.merge_from_list(config2cmdargs(config)) + if self._cfg.hpo.personalized_ss: + if isinstance(self._client_cfgs, CS.Configuration): + self._client_cfgs.merge_from_list( + config2cmdargs(config)) + else: + self._client_cfgs = CfgNode(flatten2nestdict(config)) + else: + trial_cfg.merge_from_list(config2cmdargs(config)) results = make_trial(trial_cfg, self._client_cfgs) key1, key2 = trial_cfg.hpo.metric.split('.') perfs[i] = results[key1][key2] diff --git a/federatedscope/autotune/utils.py b/federatedscope/autotune/utils.py index 271984a35..755306c49 100644 --- a/federatedscope/autotune/utils.py +++ b/federatedscope/autotune/utils.py @@ -3,6 +3,8 @@ import pandas as pd import ConfigSpace as CS +from federatedscope.core.configs.yacs_config import CfgNode + logger = logging.getLogger(__name__) @@ -246,7 +248,14 @@ def eval_in_fs(cfg, config, budget, client_cfgs=None): # Global cfg trial_cfg = cfg.clone() # specify the configuration of interest - trial_cfg.merge_from_list(config2cmdargs(config)) + if cfg.hpo.personalized_ss: + if isinstance(client_cfgs, CS.Configuration): + client_cfgs.merge_from_list(config2cmdargs(config)) + else: + client_cfgs = CfgNode(flatten2nestdict(config)) + else: + trial_cfg.merge_from_list(config2cmdargs(config)) + # specify the budget trial_cfg.merge_from_list( ["federate.total_round_num", @@ -266,6 +275,19 @@ def eval_in_fs(cfg, config, budget, client_cfgs=None): return results +def flatten2nestdict(raw_dict, delimiter='.'): + def nested_set(dic, keys, value): + for key in keys[:-1]: + dic = dic.setdefault(key, {}) + dic[keys[-1]] = value + + new_dict = dict() + for key, value in raw_dict.items(): + keys = key.split(delimiter) + nested_set(new_dict, keys, value) + return new_dict + + def config_bool2int(config): # TODO: refactor bool/str to int import copy diff --git a/federatedscope/core/configs/cfg_hpo.py b/federatedscope/core/configs/cfg_hpo.py index 8ed8e7e88..1d41cbbc8 100644 --- a/federatedscope/core/configs/cfg_hpo.py +++ b/federatedscope/core/configs/cfg_hpo.py @@ -10,6 +10,8 @@ def extend_hpo_cfg(cfg): cfg.hpo = CN() cfg.hpo.working_folder = 'hpo' cfg.hpo.ss = '' + cfg.hpo.personalized_ss = False # If True, the ss will be discrete to + # hadamard product of client's ss cfg.hpo.num_workers = 0 cfg.hpo.init_cand_num = 16 cfg.hpo.larger_better = False diff --git a/federatedscope/hpo.py b/federatedscope/hpo.py index 789c897e7..ff1a2147f 100644 --- a/federatedscope/hpo.py +++ b/federatedscope/hpo.py @@ -10,6 +10,7 @@ from federatedscope.core.auxiliaries.utils import setup_seed from federatedscope.core.auxiliaries.logging import update_logger from federatedscope.core.cmd_args import parse_args, parse_client_cfg +from federatedscope.core.auxiliaries.data_builder import get_data from federatedscope.core.configs.config import global_cfg, CfgNode from federatedscope.autotune import get_scheduler, run_scheduler @@ -42,5 +43,10 @@ else: client_cfgs = None + # Update client num + data, modified_cfg = get_data(config=init_cfg.clone(), + client_cfgs=client_cfgs) + init_cfg.merge_from_other_cfg(modified_cfg) + scheduler = get_scheduler(init_cfg, client_cfgs) run_scheduler(scheduler, init_cfg, client_cfgs) diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp.yaml new file mode 100644 index 000000000..a1956e817 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp.yaml @@ -0,0 +1,52 @@ +use_gpu: True +device: 3 +outdir: bo_gp_femnist_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: bo_gp + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' + personalized_ss: True + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_gp_femnist_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp_wrap.yaml new file mode 100644 index 000000000..0011c09b5 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp_wrap.yaml @@ -0,0 +1,63 @@ +use_gpu: True +device: 3 +outdir: bo_gp_wrap_femnist_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_bo_gp + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + table: + num: 27 + fedex: + psn: True + pi_lr: 0.01 + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_gp_wrap_femnist_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde.yaml new file mode 100644 index 000000000..aa182ad5f --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde.yaml @@ -0,0 +1,52 @@ +use_gpu: True +device: 3 +outdir: bo_kde_femnist_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: bo_kde + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' + personalized_ss: True + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_kde_femnist_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde_wrap.yaml new file mode 100644 index 000000000..16304cc6c --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde_wrap.yaml @@ -0,0 +1,63 @@ +use_gpu: True +device: 3 +outdir: bo_kde_wrap_femnist_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_bo_kde + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + table: + num: 27 + fedex: + psn: True + pi_lr: 0.01 + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_kde_wrap_femnist_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf.yaml new file mode 100644 index 000000000..b6eab5cba --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf.yaml @@ -0,0 +1,52 @@ +use_gpu: True +device: 3 +outdir: bo_rf_femnist_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: bo_rf + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' + personalized_ss: True + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_rf_femnist_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf_wrap.yaml new file mode 100644 index 000000000..b0f67756f --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf_wrap.yaml @@ -0,0 +1,63 @@ +use_gpu: True +device: 3 +outdir: bo_rf_wrap_femnist_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_bo_rf + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + table: + num: 27 + fedex: + psn: True + pi_lr: 0.01 + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_rf_wrap_femnist_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bohb.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bohb.yaml new file mode 100644 index 000000000..02613eff5 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bohb.yaml @@ -0,0 +1,52 @@ +use_gpu: True +device: 3 +outdir: bohb_femnist_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: bohb + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' + personalized_ss: True + sha: + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bohb_femnist_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bohb_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bohb_wrap.yaml new file mode 100644 index 000000000..7f8dd2044 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bohb_wrap.yaml @@ -0,0 +1,63 @@ +use_gpu: True +device: 3 +outdir: bohb_wrap_femnist_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_bohb + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' + sha: + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 + table: + num: 27 + fedex: + psn: True + pi_lr: 0.01 + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bohb_wrap_femnist_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/hb.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/hb.yaml new file mode 100644 index 000000000..132ee5f7b --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/hb.yaml @@ -0,0 +1,52 @@ +use_gpu: True +device: 3 +outdir: hb_femnist_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: hb + num_workers: 0 + ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' + personalized_ss: True + sha: + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: hb_femnist_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/hb_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/hb_wrap.yaml new file mode 100644 index 000000000..3f839e7bb --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/hb_wrap.yaml @@ -0,0 +1,63 @@ +use_gpu: True +device: 3 +outdir: hb_wrap_femnist_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_hb + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' + sha: + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 + table: + num: 27 + fedex: + psn: True + pi_lr: 0.01 + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: hb_wrap_femnist_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/rs.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/rs.yaml new file mode 100644 index 000000000..4761656c6 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/rs.yaml @@ -0,0 +1,53 @@ +use_gpu: True +device: 3 +outdir: rs_femnist_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: rs + num_workers: 0 + init_cand_num: 10 + ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' + personalized_ss: True + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: rs_femnist_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/rs_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/rs_wrap.yaml new file mode 100644 index 000000000..77ee7c327 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/rs_wrap.yaml @@ -0,0 +1,63 @@ +use_gpu: True +device: 3 +outdir: rs_wrap_femnist_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_rs + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + table: + num: 27 + fedex: + psn: True + pi_lr: 0.01 + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: rs_wrap_femnist_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml new file mode 100644 index 000000000..e7d57b0ae --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml @@ -0,0 +1,19 @@ +train.optimizer.lr: + type: float + lower: 0.01 + upper: 1.0 + log: True +train.optimizer.weight_decay: + type: float + lower: 0.0 + upper: 1.0 +model.dropout: + type: cate + choices: [0.0, 0.5] +train.local_update_steps: + type: int + lower: 1 + upper: 4 +dataloader.batch_size: + type: cate + choices: [16, 32, 64] From f4bfe3c7012d96e22ec683985c7a8dc7074de9cb Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Mon, 9 Jan 2023 18:07:13 +0800 Subject: [PATCH 10/38] add run for phpo --- .../hpo_exp_scripts/uniqueness/personalized/run.sh | 11 +++++++++++ scripts/hpo_exp_scripts/usability/pubmed/run.sh | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 scripts/hpo_exp_scripts/uniqueness/personalized/run.sh diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/run.sh b/scripts/hpo_exp_scripts/uniqueness/personalized/run.sh new file mode 100644 index 000000000..cf6bb4f7f --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/run.sh @@ -0,0 +1,11 @@ +set -e + +seed=$1 + +methods=('rs' 'rs_wrap' 'bo_gp' 'bo_gp_wrap' 'bo_kde' 'bo_kde_wrap' 'bo_rf' 'bo_rf_wrap' 'hb' 'hb_wrap' 'bohb' 'bohb_wrap') + + +for (( m=0; m<${#methods[@]}; m++ )) +do + nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/uniqueness/personalized/${methods[$m]}.yaml device $((m%4)) seed ${seed} >/dev/null 2>&1 +done \ No newline at end of file diff --git a/scripts/hpo_exp_scripts/usability/pubmed/run.sh b/scripts/hpo_exp_scripts/usability/pubmed/run.sh index be5a8deb0..483b2d638 100644 --- a/scripts/hpo_exp_scripts/usability/pubmed/run.sh +++ b/scripts/hpo_exp_scripts/usability/pubmed/run.sh @@ -7,5 +7,5 @@ methods=('rs' 'rs_wrap' 'bo_gp' 'bo_gp_wrap' 'bo_kde' 'bo_kde_wrap' 'bo_rf' 'bo_ for (( m=0; m<${#methods[@]}; m++ )) do - nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/usability/pubmed/${methods[$m]}.yaml device $((m%4)) seed ${seed}>${methods[$m]}_${seed}.log & + nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/usability/pubmed/${methods[$m]}.yaml device $((m%4)) seed ${seed} >/dev/null 2>&1 done From 37d814241380f42977152a1db4cb0581a9ddff45 Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Mon, 9 Jan 2023 18:10:59 +0800 Subject: [PATCH 11/38] add TODO --- federatedscope/autotune/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/federatedscope/autotune/utils.py b/federatedscope/autotune/utils.py index 755306c49..9eb4b841a 100644 --- a/federatedscope/autotune/utils.py +++ b/federatedscope/autotune/utils.py @@ -276,6 +276,7 @@ def eval_in_fs(cfg, config, budget, client_cfgs=None): def flatten2nestdict(raw_dict, delimiter='.'): + # TODO: delete this for the function of `arm2dict` def nested_set(dic, keys, value): for key in keys[:-1]: dic = dic.setdefault(key, {}) From 11796e88f98bd7e4878a6eee239757bab1b0dc19 Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Wed, 11 Jan 2023 17:03:33 +0800 Subject: [PATCH 12/38] add attack for fedex --- federatedscope/autotune/fedex/client.py | 8 +++ federatedscope/core/configs/cfg_hpo.py | 4 ++ .../robustness/attack/bo_gp_wrap.yaml | 64 +++++++++++++++++++ .../robustness/attack/bo_kde_wrap.yaml | 64 +++++++++++++++++++ .../robustness/attack/bo_rf_wrap.yaml | 64 +++++++++++++++++++ .../robustness/attack/bohb_wrap.yaml | 64 +++++++++++++++++++ .../robustness/attack/hb_wrap.yaml | 64 +++++++++++++++++++ .../robustness/attack/rs_wrap.yaml | 64 +++++++++++++++++++ .../hpo_exp_scripts/robustness/attack/run.sh | 13 ++++ .../hpo_exp_scripts/robustness/attack/ss.yaml | 19 ++++++ 10 files changed, 428 insertions(+) create mode 100644 scripts/hpo_exp_scripts/robustness/attack/bo_gp_wrap.yaml create mode 100644 scripts/hpo_exp_scripts/robustness/attack/bo_kde_wrap.yaml create mode 100644 scripts/hpo_exp_scripts/robustness/attack/bo_rf_wrap.yaml create mode 100644 scripts/hpo_exp_scripts/robustness/attack/bohb_wrap.yaml create mode 100644 scripts/hpo_exp_scripts/robustness/attack/hb_wrap.yaml create mode 100644 scripts/hpo_exp_scripts/robustness/attack/rs_wrap.yaml create mode 100644 scripts/hpo_exp_scripts/robustness/attack/run.sh create mode 100644 scripts/hpo_exp_scripts/robustness/attack/ss.yaml diff --git a/federatedscope/autotune/fedex/client.py b/federatedscope/autotune/fedex/client.py index a20b6682f..7d7f270b7 100644 --- a/federatedscope/autotune/fedex/client.py +++ b/federatedscope/autotune/fedex/client.py @@ -1,6 +1,7 @@ import logging import json import copy +import numpy as np from federatedscope.core.message import Message from federatedscope.core.workers import Client @@ -12,6 +13,9 @@ class FedExClient(Client): """Some code snippets are borrowed from the open-sourced FedEx ( https://github.com/mkhodak/FedEx) """ + def add_noise(self): + ... + def _apply_hyperparams(self, hyperparams): """Apply the given hyperparameters Arguments: @@ -56,6 +60,10 @@ def callback_funcs_for_model_para(self, message: Message): rnd=self.state, role='Client #{}'.format(self.ID), return_raw=True)) + # Inject + if self.ID in self._cfg.hpo.fedex.attack.id: + results['val_avg_loss_after'] += \ + self._cfg.hpo.fedex.attack.sigma * np.random.randn() results['arms'] = arms content = (sample_size, model_para_all, results) diff --git a/federatedscope/core/configs/cfg_hpo.py b/federatedscope/core/configs/cfg_hpo.py index 1d41cbbc8..5dede2e7c 100644 --- a/federatedscope/core/configs/cfg_hpo.py +++ b/federatedscope/core/configs/cfg_hpo.py @@ -44,6 +44,10 @@ def extend_hpo_cfg(cfg): cfg.hpo.fedex.gamma = .0 cfg.hpo.fedex.diff = False + cfg.hpo.fedex.attack = CN() + cfg.hpo.fedex.attack.id = [] # client IDs who inject noise into policy + cfg.hpo.fedex.attack.sigma = 1.0 # sigma of white noise + # Table cfg.hpo.table = CN() cfg.hpo.table.eps = 0.1 diff --git a/scripts/hpo_exp_scripts/robustness/attack/bo_gp_wrap.yaml b/scripts/hpo_exp_scripts/robustness/attack/bo_gp_wrap.yaml new file mode 100644 index 000000000..92a06cbe2 --- /dev/null +++ b/scripts/hpo_exp_scripts/robustness/attack/bo_gp_wrap.yaml @@ -0,0 +1,64 @@ +use_gpu: True +device: 3 +outdir: bo_gp_wrap_femnist_avg_robust +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_bo_gp + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/robustness/attack/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + table: + num: 27 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + attack: + id: [ 1 ] + sigma: 1.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_gp_wrap_femnist_avg_robust diff --git a/scripts/hpo_exp_scripts/robustness/attack/bo_kde_wrap.yaml b/scripts/hpo_exp_scripts/robustness/attack/bo_kde_wrap.yaml new file mode 100644 index 000000000..c66642752 --- /dev/null +++ b/scripts/hpo_exp_scripts/robustness/attack/bo_kde_wrap.yaml @@ -0,0 +1,64 @@ +use_gpu: True +device: 3 +outdir: bo_kde_wrap_femnist_avg_robust +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_bo_kde + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/robustness/attack/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + table: + num: 27 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + attack: + id: [ 1 ] + sigma: 1.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_kde_wrap_femnist_avg_robust diff --git a/scripts/hpo_exp_scripts/robustness/attack/bo_rf_wrap.yaml b/scripts/hpo_exp_scripts/robustness/attack/bo_rf_wrap.yaml new file mode 100644 index 000000000..906775e5e --- /dev/null +++ b/scripts/hpo_exp_scripts/robustness/attack/bo_rf_wrap.yaml @@ -0,0 +1,64 @@ +use_gpu: True +device: 3 +outdir: bo_rf_wrap_femnist_avg_robust +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_bo_rf + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/robustness/attack/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + table: + num: 27 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + attack: + id: [ 1 ] + sigma: 1.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bo_rf_wrap_femnist_avg_robust diff --git a/scripts/hpo_exp_scripts/robustness/attack/bohb_wrap.yaml b/scripts/hpo_exp_scripts/robustness/attack/bohb_wrap.yaml new file mode 100644 index 000000000..720356da7 --- /dev/null +++ b/scripts/hpo_exp_scripts/robustness/attack/bohb_wrap.yaml @@ -0,0 +1,64 @@ +use_gpu: True +device: 3 +outdir: bohb_wrap_femnist_avg_robust +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_bohb + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/robustness/attack/ss.yaml' + sha: + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 + table: + num: 27 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + attack: + id: [ 1 ] + sigma: 1.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: bohb_wrap_femnist_avg_robust diff --git a/scripts/hpo_exp_scripts/robustness/attack/hb_wrap.yaml b/scripts/hpo_exp_scripts/robustness/attack/hb_wrap.yaml new file mode 100644 index 000000000..b91fa7f15 --- /dev/null +++ b/scripts/hpo_exp_scripts/robustness/attack/hb_wrap.yaml @@ -0,0 +1,64 @@ +use_gpu: True +device: 3 +outdir: hb_wrap_femnist_avg_robust +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_hb + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/robustness/attack/ss.yaml' + sha: + budgets: [ 9, 81 ] + elim_rate: 3 + iter: 12 + table: + num: 27 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + attack: + id: [ 1 ] + sigma: 1.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: hb_wrap_femnist_avg_robust diff --git a/scripts/hpo_exp_scripts/robustness/attack/rs_wrap.yaml b/scripts/hpo_exp_scripts/robustness/attack/rs_wrap.yaml new file mode 100644 index 000000000..8a332bcf4 --- /dev/null +++ b/scripts/hpo_exp_scripts/robustness/attack/rs_wrap.yaml @@ -0,0 +1,64 @@ +use_gpu: True +device: 3 +outdir: rs_wrap_femnist_avg_robust +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: wrap_rs + num_workers: 0 + init_cand_num: 100 + ss: 'scripts/hpo_exp_scripts/robustness/attack/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + table: + num: 27 + fedex: + sched: 'aggressive' + use: True + diff: False + eta0: -1.0 + gamma: 0.0 + attack: + id: [ 1 ] + sigma: 1.0 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: rs_wrap_femnist_avg_robust diff --git a/scripts/hpo_exp_scripts/robustness/attack/run.sh b/scripts/hpo_exp_scripts/robustness/attack/run.sh new file mode 100644 index 000000000..8cad43cab --- /dev/null +++ b/scripts/hpo_exp_scripts/robustness/attack/run.sh @@ -0,0 +1,13 @@ +set -e + +attacker=$1 +sigma=$2 +seed=$3 + +methods=('rs_wrap' 'bo_gp_wrap' 'bo_kde_wrap' 'bo_rf_wrap' 'hb_wrap' 'bohb_wrap') + + +for (( m=0; m<${#methods[@]}; m++ )) +do + nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/uniqueness/cifar10/${methods[$m]}.yaml device $((m%4)) seed ${seed} hpo.fedex.attack.id ${attacker} hpo.fedex.attack.sigma ${sigma} >/dev/null 2>&1 & +done \ No newline at end of file diff --git a/scripts/hpo_exp_scripts/robustness/attack/ss.yaml b/scripts/hpo_exp_scripts/robustness/attack/ss.yaml new file mode 100644 index 000000000..e7d57b0ae --- /dev/null +++ b/scripts/hpo_exp_scripts/robustness/attack/ss.yaml @@ -0,0 +1,19 @@ +train.optimizer.lr: + type: float + lower: 0.01 + upper: 1.0 + log: True +train.optimizer.weight_decay: + type: float + lower: 0.0 + upper: 1.0 +model.dropout: + type: cate + choices: [0.0, 0.5] +train.local_update_steps: + type: int + lower: 1 + upper: 4 +dataloader.batch_size: + type: cate + choices: [16, 32, 64] From 7800fff7313aed268d2efc9fccaec951c066ec6a Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Wed, 11 Jan 2023 20:28:41 +0800 Subject: [PATCH 13/38] fix minor bugs --- federatedscope/autotune/utils.py | 3 ++- federatedscope/hpo.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/federatedscope/autotune/utils.py b/federatedscope/autotune/utils.py index 9eb4b841a..2787ceed1 100644 --- a/federatedscope/autotune/utils.py +++ b/federatedscope/autotune/utils.py @@ -41,7 +41,7 @@ def parse_condition_param(condition, ss): f'should be in' \ f' {str_func_mapping.keys()}.' - if cond_type in ['and', 'in']: + if cond_type in ['and', 'in', 'or']: return str_func_mapping[cond_type]( parse_condition_param(condition['child'], ss), parse_condition_param(condition['parent'], ss), @@ -96,6 +96,7 @@ def parse_search_space(config_path): if name.startswith('condition'): conditions.append(parse_condition_param(raw_ss_config[name], ss)) ss.add_conditions(conditions) + print(ss) return ss diff --git a/federatedscope/hpo.py b/federatedscope/hpo.py index ff1a2147f..02bbfb23c 100644 --- a/federatedscope/hpo.py +++ b/federatedscope/hpo.py @@ -49,4 +49,7 @@ init_cfg.merge_from_other_cfg(modified_cfg) scheduler = get_scheduler(init_cfg, client_cfgs) + print(scheduler._search_space) + raise ValueError + run_scheduler(scheduler, init_cfg, client_cfgs) From 6513551b15a7551b6d1504916cbff8472f41d9e6 Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Wed, 11 Jan 2023 20:29:43 +0800 Subject: [PATCH 14/38] rm print --- federatedscope/hpo.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/federatedscope/hpo.py b/federatedscope/hpo.py index 02bbfb23c..ff1a2147f 100644 --- a/federatedscope/hpo.py +++ b/federatedscope/hpo.py @@ -49,7 +49,4 @@ init_cfg.merge_from_other_cfg(modified_cfg) scheduler = get_scheduler(init_cfg, client_cfgs) - print(scheduler._search_space) - raise ValueError - run_scheduler(scheduler, init_cfg, client_cfgs) From be46716316be8769464d0f990145b7f0a31aca9a Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Wed, 11 Jan 2023 20:30:04 +0800 Subject: [PATCH 15/38] rm print --- federatedscope/autotune/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/federatedscope/autotune/utils.py b/federatedscope/autotune/utils.py index 2787ceed1..fe67c21cc 100644 --- a/federatedscope/autotune/utils.py +++ b/federatedscope/autotune/utils.py @@ -96,7 +96,6 @@ def parse_search_space(config_path): if name.startswith('condition'): conditions.append(parse_condition_param(raw_ss_config[name], ss)) ss.add_conditions(conditions) - print(ss) return ss From d5f1901e5bf4977ac3900780a23020bfc4d020fb Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Thu, 12 Jan 2023 16:27:52 +0800 Subject: [PATCH 16/38] add draw scipts --- scripts/hpo_exp_scripts/draw_traj.py | 128 +++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 scripts/hpo_exp_scripts/draw_traj.py diff --git a/scripts/hpo_exp_scripts/draw_traj.py b/scripts/hpo_exp_scripts/draw_traj.py new file mode 100644 index 000000000..5c201c424 --- /dev/null +++ b/scripts/hpo_exp_scripts/draw_traj.py @@ -0,0 +1,128 @@ +METHOD = { + 'rs': 'RS', + 'bo_gp': 'BO_GP', + 'bo_rf': 'BO_RF', + 'bo_kde': 'BO_KDE', + 'hb': 'HB', + 'bohb': 'BOHB', + 'rs_wrap': 'RS+FedEx', + 'bo_gp_wrap': 'BO_GP+FedEx', + 'bo_rf_wrap': 'BO_RF+FedEx', + 'bo_kde_wrap': 'BO_KDE+FedEx', + 'hb_wrap': 'HB+FedEx', + 'bohb_wrap': 'BOHB+FedEx', +} + +COLORS = [ + u'#1f77b4', u'#ff7f0e', u'#2ca02c', u'#d62728', u'#8c564b', u'#9467bd' +] + + +def parse_logs(root, + file_list, + eval_key='client_summarized_weighted_avg', + suf=''): + import os + import numpy as np + import matplotlib.pyplot as plt + from tqdm import tqdm + + FONTSIZE = 40 + MARKSIZE = 25 + + def process(file): + history = [] + with open(file, 'r') as F: + F = F.readlines() + for idx, line in tqdm(enumerate(F)): + if "'Round': 'Final'" in line: + last_line = F[idx - 2] + _, last_line = last_line.split('INFO: ') + last_line = eval(last_line) + config = {'federate.total_round_num': last_line['Round']} + try: + state, line = line.split('INFO: ') + results = eval(line) + performance = results['Results_raw'][eval_key][ + 'val_avg_loss'] + history.append((config, performance)) + except Exception as error: + continue + best_seen = np.inf + tol_budget = 0 + x, y = [], [] + + for config, performance in history: + tol_budget += config['federate.total_round_num'] + if best_seen > performance or config[ + 'federate.total_round_num'] > tmp_b: + best_seen = performance + x.append(tol_budget) + y.append(best_seen) + tmp_b = config['federate.total_round_num'] + return np.array(x) / tol_budget, np.array(y) + + # Draw + plt.figure(figsize=(10, 7.5)) + plt.xticks(fontsize=FONTSIZE) + plt.yticks(fontsize=FONTSIZE) + + plt.xlabel('Fraction of budget', size=FONTSIZE) + plt.ylabel('Loss', size=FONTSIZE) + + data_x = {} + data_y = {} + for file in file_list: + data_x[file] = [] + data_y[file] = [] + + for prefix in root: + for file in file_list: + if 'wrap' in file: + new_name = f'wrap_{file[:-5]}' + else: + new_name = file + + if 'hb' in file: + budget = [9, 81] + else: + budget = [50, 50] + x, y = process( + os.path.join( + prefix, f'{file}{suf}', + f'{new_name}_{budget}_server_global_eval.val_avg_loss', + 'exp_print.log')) + data_x[file].append(x) + data_y[file].append(y) + + for i, file in enumerate(file_list): + if 'wrap' in file: + linestyle = '--' + else: + linestyle = '-' + plt.plot(np.mean(data_x[file], axis=0), + np.mean(data_y[file], axis=0), + linewidth=1, + color=COLORS[i % len(COLORS)], + markersize=MARKSIZE, + linestyle=linestyle) + if file_list[0].endswith('_opt.log'): + suffix = 'opt' + else: + suffix = 'avg' + plt.xscale("log") + plt.xticks([0.01, 0.1, 1], ['1e-2', '1e-1', '1']) + plt.legend(list(METHOD.values()), + loc='upper right', + prop={'size': 22}, + bbox_to_anchor=(1.5, 1), + borderaxespad=0) + plt.savefig(f'exp2_{suffix}.pdf', bbox_inches='tight') + plt.show() + plt.close() + + +parse_logs(['t0', 't1', 't2'], + list(METHOD.keys()), + eval_key='server_global_eval', + suf='_pubmed_avg') From cf17681b3dd0580a672a2765d48059a9102fa2bc Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Thu, 12 Jan 2023 18:02:33 +0800 Subject: [PATCH 17/38] remove extra line --- federatedscope/autotune/fedex/client.py | 3 --- federatedscope/autotune/utils.py | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/federatedscope/autotune/fedex/client.py b/federatedscope/autotune/fedex/client.py index 7d7f270b7..847b2113d 100644 --- a/federatedscope/autotune/fedex/client.py +++ b/federatedscope/autotune/fedex/client.py @@ -13,9 +13,6 @@ class FedExClient(Client): """Some code snippets are borrowed from the open-sourced FedEx ( https://github.com/mkhodak/FedEx) """ - def add_noise(self): - ... - def _apply_hyperparams(self, hyperparams): """Apply the given hyperparameters Arguments: diff --git a/federatedscope/autotune/utils.py b/federatedscope/autotune/utils.py index fe67c21cc..8c9e04cbc 100644 --- a/federatedscope/autotune/utils.py +++ b/federatedscope/autotune/utils.py @@ -65,6 +65,7 @@ def parse_search_space(config_path): """ + # TODO: add seed for `ConfigurationSpace` ss = CS.ConfigurationSpace() conditions = [] From 1b899cc70aff82b8326600c38cca424397c9094b Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Thu, 12 Jan 2023 18:30:27 +0800 Subject: [PATCH 18/38] add suffix --- scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp.yaml | 4 ++-- .../hpo_exp_scripts/uniqueness/personalized/bo_gp_wrap.yaml | 4 ++-- scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde.yaml | 4 ++-- .../hpo_exp_scripts/uniqueness/personalized/bo_kde_wrap.yaml | 4 ++-- scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf.yaml | 4 ++-- .../hpo_exp_scripts/uniqueness/personalized/bo_rf_wrap.yaml | 4 ++-- scripts/hpo_exp_scripts/uniqueness/personalized/bohb.yaml | 4 ++-- .../hpo_exp_scripts/uniqueness/personalized/bohb_wrap.yaml | 4 ++-- scripts/hpo_exp_scripts/uniqueness/personalized/hb.yaml | 4 ++-- scripts/hpo_exp_scripts/uniqueness/personalized/hb_wrap.yaml | 4 ++-- scripts/hpo_exp_scripts/uniqueness/personalized/rs.yaml | 4 ++-- scripts/hpo_exp_scripts/uniqueness/personalized/rs_wrap.yaml | 4 ++-- 12 files changed, 24 insertions(+), 24 deletions(-) diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp.yaml index a1956e817..e74b3bb68 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp.yaml @@ -1,6 +1,6 @@ use_gpu: True device: 3 -outdir: bo_gp_femnist_avg +outdir: bo_gp_femnist_avg_per early_stop: patience: 100 seed: 12345 @@ -49,4 +49,4 @@ hpo: elim_rate: 3 iter: 50 metric: 'client_summarized_weighted_avg.val_avg_loss' - working_folder: bo_gp_femnist_avg + working_folder: bo_gp_femnist_avg_per diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp_wrap.yaml index 0011c09b5..ec1d583a9 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp_wrap.yaml @@ -1,6 +1,6 @@ use_gpu: True device: 3 -outdir: bo_gp_wrap_femnist_avg +outdir: bo_gp_wrap_femnist_avg_per early_stop: patience: 100 seed: 12345 @@ -60,4 +60,4 @@ hpo: eta0: -1.0 gamma: 0.0 metric: 'client_summarized_weighted_avg.val_avg_loss' - working_folder: bo_gp_wrap_femnist_avg + working_folder: bo_gp_wrap_femnist_avg_per diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde.yaml index aa182ad5f..ac137084a 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde.yaml @@ -1,6 +1,6 @@ use_gpu: True device: 3 -outdir: bo_kde_femnist_avg +outdir: bo_kde_femnist_avg_per early_stop: patience: 100 seed: 12345 @@ -49,4 +49,4 @@ hpo: elim_rate: 3 iter: 50 metric: 'client_summarized_weighted_avg.val_avg_loss' - working_folder: bo_kde_femnist_avg + working_folder: bo_kde_femnist_avg_per diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde_wrap.yaml index 16304cc6c..d6e3e2c62 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde_wrap.yaml @@ -1,6 +1,6 @@ use_gpu: True device: 3 -outdir: bo_kde_wrap_femnist_avg +outdir: bo_kde_wrap_femnist_avg_per early_stop: patience: 100 seed: 12345 @@ -60,4 +60,4 @@ hpo: eta0: -1.0 gamma: 0.0 metric: 'client_summarized_weighted_avg.val_avg_loss' - working_folder: bo_kde_wrap_femnist_avg + working_folder: bo_kde_wrap_femnist_avg_per diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf.yaml index b6eab5cba..783fb28f0 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf.yaml @@ -1,6 +1,6 @@ use_gpu: True device: 3 -outdir: bo_rf_femnist_avg +outdir: bo_rf_femnist_avg_per early_stop: patience: 100 seed: 12345 @@ -49,4 +49,4 @@ hpo: elim_rate: 3 iter: 50 metric: 'client_summarized_weighted_avg.val_avg_loss' - working_folder: bo_rf_femnist_avg + working_folder: bo_rf_femnist_avg_per diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf_wrap.yaml index b0f67756f..165ee1a1f 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf_wrap.yaml @@ -1,6 +1,6 @@ use_gpu: True device: 3 -outdir: bo_rf_wrap_femnist_avg +outdir: bo_rf_wrap_femnist_avg_per early_stop: patience: 100 seed: 12345 @@ -60,4 +60,4 @@ hpo: eta0: -1.0 gamma: 0.0 metric: 'client_summarized_weighted_avg.val_avg_loss' - working_folder: bo_rf_wrap_femnist_avg + working_folder: bo_rf_wrap_femnist_avg_per diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bohb.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bohb.yaml index 02613eff5..d18031101 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/bohb.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bohb.yaml @@ -1,6 +1,6 @@ use_gpu: True device: 3 -outdir: bohb_femnist_avg +outdir: bohb_femnist_avg_per early_stop: patience: 100 seed: 12345 @@ -49,4 +49,4 @@ hpo: elim_rate: 3 iter: 12 metric: 'client_summarized_weighted_avg.val_avg_loss' - working_folder: bohb_femnist_avg + working_folder: bohb_femnist_avg_per diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bohb_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bohb_wrap.yaml index 7f8dd2044..59a9089b3 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/bohb_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bohb_wrap.yaml @@ -1,6 +1,6 @@ use_gpu: True device: 3 -outdir: bohb_wrap_femnist_avg +outdir: bohb_wrap_femnist_avg_per early_stop: patience: 100 seed: 12345 @@ -60,4 +60,4 @@ hpo: eta0: -1.0 gamma: 0.0 metric: 'client_summarized_weighted_avg.val_avg_loss' - working_folder: bohb_wrap_femnist_avg + working_folder: bohb_wrap_femnist_avg_per diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/hb.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/hb.yaml index 132ee5f7b..f50553dca 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/hb.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/hb.yaml @@ -1,6 +1,6 @@ use_gpu: True device: 3 -outdir: hb_femnist_avg +outdir: hb_femnist_avg_per early_stop: patience: 100 seed: 12345 @@ -49,4 +49,4 @@ hpo: elim_rate: 3 iter: 12 metric: 'client_summarized_weighted_avg.val_avg_loss' - working_folder: hb_femnist_avg + working_folder: hb_femnist_avg_per diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/hb_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/hb_wrap.yaml index 3f839e7bb..94771f381 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/hb_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/hb_wrap.yaml @@ -1,6 +1,6 @@ use_gpu: True device: 3 -outdir: hb_wrap_femnist_avg +outdir: hb_wrap_femnist_avg_per early_stop: patience: 100 seed: 12345 @@ -60,4 +60,4 @@ hpo: eta0: -1.0 gamma: 0.0 metric: 'client_summarized_weighted_avg.val_avg_loss' - working_folder: hb_wrap_femnist_avg + working_folder: hb_wrap_femnist_avg_per diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/rs.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/rs.yaml index 4761656c6..06a994890 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/rs.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/rs.yaml @@ -1,6 +1,6 @@ use_gpu: True device: 3 -outdir: rs_femnist_avg +outdir: rs_femnist_avg_per early_stop: patience: 100 seed: 12345 @@ -50,4 +50,4 @@ hpo: elim_rate: 3 iter: 50 metric: 'client_summarized_weighted_avg.val_avg_loss' - working_folder: rs_femnist_avg + working_folder: rs_femnist_avg_per diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/rs_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/rs_wrap.yaml index 77ee7c327..24860791a 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/rs_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/rs_wrap.yaml @@ -1,6 +1,6 @@ use_gpu: True device: 3 -outdir: rs_wrap_femnist_avg +outdir: rs_wrap_femnist_avg_per early_stop: patience: 100 seed: 12345 @@ -60,4 +60,4 @@ hpo: eta0: -1.0 gamma: 0.0 metric: 'client_summarized_weighted_avg.val_avg_loss' - working_folder: rs_wrap_femnist_avg + working_folder: rs_wrap_femnist_avg_per From a434410ac59adcdb95ac26eb1c2250abe29aad33 Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Thu, 12 Jan 2023 19:19:51 +0800 Subject: [PATCH 19/38] fix scripts --- scripts/hpo_exp_scripts/robustness/attack/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/hpo_exp_scripts/robustness/attack/run.sh b/scripts/hpo_exp_scripts/robustness/attack/run.sh index 8cad43cab..49f80f9e4 100644 --- a/scripts/hpo_exp_scripts/robustness/attack/run.sh +++ b/scripts/hpo_exp_scripts/robustness/attack/run.sh @@ -9,5 +9,5 @@ methods=('rs_wrap' 'bo_gp_wrap' 'bo_kde_wrap' 'bo_rf_wrap' 'hb_wrap' 'bohb_wrap' for (( m=0; m<${#methods[@]}; m++ )) do - nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/uniqueness/cifar10/${methods[$m]}.yaml device $((m%4)) seed ${seed} hpo.fedex.attack.id ${attacker} hpo.fedex.attack.sigma ${sigma} >/dev/null 2>&1 & + nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/robustness/attack/${methods[$m]}.yaml device $((m%4)) seed ${seed} hpo.fedex.attack.id ${attacker} hpo.fedex.attack.sigma ${sigma} >/dev/null 2>&1 & done \ No newline at end of file From 700c2b92060c9d8a47dd2eeab6bba6f9e74f6478 Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Thu, 12 Jan 2023 19:59:27 +0800 Subject: [PATCH 20/38] add space --- scripts/hpo_exp_scripts/robustness/attack/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/hpo_exp_scripts/robustness/attack/run.sh b/scripts/hpo_exp_scripts/robustness/attack/run.sh index 49f80f9e4..7dd2eca42 100644 --- a/scripts/hpo_exp_scripts/robustness/attack/run.sh +++ b/scripts/hpo_exp_scripts/robustness/attack/run.sh @@ -10,4 +10,4 @@ methods=('rs_wrap' 'bo_gp_wrap' 'bo_kde_wrap' 'bo_rf_wrap' 'hb_wrap' 'bohb_wrap' for (( m=0; m<${#methods[@]}; m++ )) do nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/robustness/attack/${methods[$m]}.yaml device $((m%4)) seed ${seed} hpo.fedex.attack.id ${attacker} hpo.fedex.attack.sigma ${sigma} >/dev/null 2>&1 & -done \ No newline at end of file +done From 56d171d4b86c58cb158088d5919d4e3c325f6bf9 Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Thu, 12 Jan 2023 20:16:49 +0800 Subject: [PATCH 21/38] modify device num --- scripts/hpo_exp_scripts/robustness/attack/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/hpo_exp_scripts/robustness/attack/run.sh b/scripts/hpo_exp_scripts/robustness/attack/run.sh index 7dd2eca42..1ab004ef1 100644 --- a/scripts/hpo_exp_scripts/robustness/attack/run.sh +++ b/scripts/hpo_exp_scripts/robustness/attack/run.sh @@ -9,5 +9,5 @@ methods=('rs_wrap' 'bo_gp_wrap' 'bo_kde_wrap' 'bo_rf_wrap' 'hb_wrap' 'bohb_wrap' for (( m=0; m<${#methods[@]}; m++ )) do - nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/robustness/attack/${methods[$m]}.yaml device $((m%4)) seed ${seed} hpo.fedex.attack.id ${attacker} hpo.fedex.attack.sigma ${sigma} >/dev/null 2>&1 & + nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/robustness/attack/${methods[$m]}.yaml device $((m%8)) seed ${seed} hpo.fedex.attack.id ${attacker} hpo.fedex.attack.sigma ${sigma} >/dev/null 2>&1 & done From 55a26f50a466bd81425e4e611231ffc1ea012586 Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Fri, 13 Jan 2023 16:29:11 +0800 Subject: [PATCH 22/38] refactor fedex --- federatedscope/autotune/algos.py | 172 +++++++++++++------------ federatedscope/autotune/hpbandster.py | 40 +++--- federatedscope/autotune/run.py | 3 +- federatedscope/autotune/smac.py | 11 +- federatedscope/autotune/utils.py | 72 ++++++++++- federatedscope/core/configs/README.md | 9 +- federatedscope/core/configs/cfg_hpo.py | 11 +- 7 files changed, 191 insertions(+), 127 deletions(-) diff --git a/federatedscope/autotune/algos.py b/federatedscope/autotune/algos.py index b2a05b6e8..9d397e9c3 100644 --- a/federatedscope/autotune/algos.py +++ b/federatedscope/autotune/algos.py @@ -72,14 +72,16 @@ def get_scheduler(init_cfg, client_cfgs=None): client_cfgs: client-specific configuration """ - if init_cfg.hpo.scheduler in [ - 'sha', 'rs', 'bo_kde', 'bohb', 'hb', 'bo_gp', 'bo_rf' - ]: - scheduler = SuccessiveHalvingAlgo(init_cfg, client_cfgs) + # TODO: fix wrap_sha + support_optimizer = [ + 'sha', 'rs', 'bo_kde', 'hb', 'bohb', 'wrap_rs', 'wrap_bo_kde', + 'wrap_hb', 'wrap_bohb', 'bo_gp', 'bo_rf', 'wrap_bo_gp', 'wrap_bo_rf' + ] + assert init_cfg.hpo.scheduler in support_optimizer, \ + f'`init_cfg.hpo.scheduler` must be one of {support_optimizer}.' + scheduler = SuccessiveHalvingAlgo(init_cfg, client_cfgs) # elif init_cfg.hpo.scheduler == 'pbt': # scheduler = PBT(init_cfg) - elif init_cfg.hpo.scheduler.startswith('wrap', client_cfgs): - scheduler = SHAWrapFedex(init_cfg) return scheduler @@ -353,85 +355,85 @@ def _generate_next_population(self, configs, perfs): return next_population -class SHAWrapFedex(SuccessiveHalvingAlgo): - """This SHA is customized as a wrapper for FedEx algorithm.""" - def _make_local_perturbation(self, config): - neighbor = dict() - for k in config: - if 'fedex' in k or 'fedopt' in k or k in [ - 'federate.save_to', 'federate.total_round_num', 'eval.freq' - ]: - # a workaround - continue - hyper = self._search_space.get(k) - if isinstance(hyper, CS.UniformFloatHyperparameter): - lb, ub = hyper.lower, hyper.upper - diameter = self._cfg.hpo.table.eps * (ub - lb) - new_val = (config[k] - - 0.5 * diameter) + np.random.uniform() * diameter - neighbor[k] = float(np.clip(new_val, lb, ub)) - elif isinstance(hyper, CS.UniformIntegerHyperparameter): - lb, ub = hyper.lower, hyper.upper - diameter = self._cfg.hpo.table.eps * (ub - lb) - new_val = round( - float((config[k] - 0.5 * diameter) + - np.random.uniform() * diameter)) - neighbor[k] = int(np.clip(new_val, lb, ub)) - elif isinstance(hyper, CS.CategoricalHyperparameter): - if len(hyper.choices) == 1: - neighbor[k] = config[k] - else: - threshold = self._cfg.hpo.table.eps * len( - hyper.choices) / (len(hyper.choices) - 1) - rn = np.random.uniform() - new_val = np.random.choice( - hyper.choices) if rn <= threshold else config[k] - if type(new_val) in [np.int32, np.int64]: - neighbor[k] = int(new_val) - elif type(new_val) in [np.float32, np.float64]: - neighbor[k] = float(new_val) - else: - neighbor[k] = str(new_val) - else: - raise TypeError("Value of {} has an invalid type {}".format( - k, type(config[k]))) - - return neighbor - - def _setup(self): - # self._cache_yaml() - init_configs = super(SHAWrapFedex, self)._setup() - new_init_configs = [] - for idx, trial_cfg in enumerate(init_configs): - arms = dict(("arm{}".format(1 + j), - self._make_local_perturbation(trial_cfg)) - for j in range(self._cfg.hpo.table.num - 1)) - arms['arm0'] = dict( - (k, v) for k, v in trial_cfg.items() if k in arms['arm1']) - with open( - os.path.join(self._cfg.hpo.working_folder, - f'{idx}_tmp_grid_search_space.yaml'), - 'w') as f: - yaml.dump(arms, f) - new_trial_cfg = dict() - for k in trial_cfg: - if k not in arms['arm0']: - new_trial_cfg[k] = trial_cfg[k] - new_trial_cfg['hpo.table.idx'] = idx - new_trial_cfg['hpo.fedex.ss'] = os.path.join( - self._cfg.hpo.working_folder, - f"{new_trial_cfg['hpo.table.idx']}_tmp_grid_search_space.yaml") - new_trial_cfg['federate.save_to'] = os.path.join( - self._cfg.hpo.working_folder, "idx_{}.pth".format(idx)) - new_init_configs.append(new_trial_cfg) - - self._search_space.add_hyperparameter( - CS.CategoricalHyperparameter("hpo.table.idx", - choices=list( - range(len(new_init_configs))))) - - return new_init_configs - +# class SHAWrapFedex(SuccessiveHalvingAlgo): +# """This SHA is customized as a wrapper for FedEx algorithm.""" +# def _make_local_perturbation(self, config): +# neighbor = dict() +# for k in config: +# if 'fedex' in k or 'fedopt' in k or k in [ +# 'federate.save_to', 'federate.total_round_num', +# 'eval.freq' +# ]: +# # a workaround +# continue +# hyper = self._search_space.get(k) +# if isinstance(hyper, CS.UniformFloatHyperparameter): +# lb, ub = hyper.lower, hyper.upper +# diameter = self._cfg.fedex.wrapper.eps * (ub - lb) +# new_val = (config[k] - +# 0.5 * diameter) + np.random.uniform() * diameter +# neighbor[k] = float(np.clip(new_val, lb, ub)) +# elif isinstance(hyper, CS.UniformIntegerHyperparameter): +# lb, ub = hyper.lower, hyper.upper +# diameter = self._cfg.fedex.wrapper.eps * (ub - lb) +# new_val = round( +# float((config[k] - 0.5 * diameter) + +# np.random.uniform() * diameter)) +# neighbor[k] = int(np.clip(new_val, lb, ub)) +# elif isinstance(hyper, CS.CategoricalHyperparameter): +# if len(hyper.choices) == 1: +# neighbor[k] = config[k] +# else: +# threshold = self._cfg.fedex.wrapper.eps * len( +# hyper.choices) / (len(hyper.choices) - 1) +# rn = np.random.uniform() +# new_val = np.random.choice( +# hyper.choices) if rn <= threshold else config[k] +# if type(new_val) in [np.int32, np.int64]: +# neighbor[k] = int(new_val) +# elif type(new_val) in [np.float32, np.float64]: +# neighbor[k] = float(new_val) +# else: +# neighbor[k] = str(new_val) +# else: +# raise TypeError("Value of {} has an invalid type {}".format( +# k, type(config[k]))) +# +# return neighbor +# +# def _setup(self): +# # self._cache_yaml() +# init_configs = super(SHAWrapFedex, self)._setup() +# new_init_configs = [] +# for idx, trial_cfg in enumerate(init_configs): +# arms = dict(("arm{}".format(1 + j), +# self._make_local_perturbation(trial_cfg)) +# for j in range(self._cfg.hpo.fedex.wrapper.arm - 1)) +# arms['arm0'] = dict( +# (k, v) for k, v in trial_cfg.items() if k in arms['arm1']) +# with open( +# os.path.join(self._cfg.hpo.working_folder, +# f'{idx}_tmp_grid_search_space.yaml'), +# 'w') as f: +# yaml.dump(arms, f) +# new_trial_cfg = dict() +# for k in trial_cfg: +# if k not in arms['arm0']: +# new_trial_cfg[k] = trial_cfg[k] +# new_trial_cfg['hpo.table.idx'] = idx +# new_trial_cfg['hpo.fedex.ss'] = os.path.join( +# self._cfg.hpo.working_folder, +# f"{new_trial_cfg['hpo.table.idx']}_tmp_grid_search_space.yaml") +# new_trial_cfg['federate.save_to'] = os.path.join( +# self._cfg.hpo.working_folder, "idx_{}.pth".format(idx)) +# new_init_configs.append(new_trial_cfg) +# +# self._search_space.add_hyperparameter( +# CS.CategoricalHyperparameter("hpo.table.idx", +# choices=list( +# range(len(new_init_configs))))) +# +# return new_init_configs # TODO: refactor PBT to enable async parallel # class PBT(IterativeScheduler): diff --git a/federatedscope/autotune/hpbandster.py b/federatedscope/autotune/hpbandster.py index a0bda9e8b..26415b507 100644 --- a/federatedscope/autotune/hpbandster.py +++ b/federatedscope/autotune/hpbandster.py @@ -14,12 +14,11 @@ logging.basicConfig(level=logging.WARNING) logger = logging.getLogger(__name__) - -def clear_cache(working_folder): - # Clear cached ckpt - for name in os.listdir(working_folder): - if name.endswith('.pth'): - os.remove(osp(working_folder, name)) +# def clear_cache(working_folder): +# # Clear cached ckpt +# for name in os.listdir(working_folder): +# if name.endswith('.pth'): +# os.remove(osp(working_folder, name)) class MyRandomSearch(RandomSearch): @@ -33,11 +32,11 @@ def __init__(self, working_folder, **kwargs): self.working_folder = working_folder super(MyBOHB, self).__init__(**kwargs) - def get_next_iteration(self, iteration, iteration_kwargs={}): - if os.path.exists(self.working_folder): - clear_cache(self.working_folder) - return super(MyBOHB, self).get_next_iteration(iteration, - iteration_kwargs) + # def get_next_iteration(self, iteration, iteration_kwargs={}): + # if os.path.exists(self.working_folder): + # clear_cache(self.working_folder) + # return super(MyBOHB, self).get_next_iteration(iteration, + # iteration_kwargs) class MyHyperBand(HyperBand): @@ -45,11 +44,11 @@ def __init__(self, working_folder, **kwargs): self.working_folder = working_folder super(MyHyperBand, self).__init__(**kwargs) - def get_next_iteration(self, iteration, iteration_kwargs={}): - if os.path.exists(self.working_folder): - clear_cache(self.working_folder) - return super(MyHyperBand, - self).get_next_iteration(iteration, iteration_kwargs) + # def get_next_iteration(self, iteration, iteration_kwargs={}): + # if os.path.exists(self.working_folder): + # clear_cache(self.working_folder) + # return super(MyHyperBand, + # self).get_next_iteration(iteration, iteration_kwargs) class MyWorker(Worker): @@ -68,8 +67,9 @@ def __init__(self, self._init_configs = [] self._perfs = [] - def compute(self, config, budget, **kwargs): - results = eval_in_fs(self.cfg, config, int(budget), self.client_cfgs) + def compute(self, config, budget, config_id, **kwargs): + results = eval_in_fs(self.cfg, config, int(budget), config_id, + self._ss, self.client_cfgs) key1, key2 = self.cfg.hpo.metric.split('.') res = results[key1][key2] config = dict(config) @@ -100,10 +100,6 @@ def summarize(self): def run_hpbandster(cfg, scheduler, client_cfgs=None): config_space = scheduler._search_space - if cfg.hpo.scheduler.startswith('wrap_'): - ss = CS.ConfigurationSpace() - ss.add_hyperparameter(config_space['hpo.table.idx']) - config_space = ss NS = hpns.NameServer(run_id=cfg.hpo.scheduler, host='127.0.0.1', port=0) ns_host, ns_port = NS.start() w = MyWorker(sleep_interval=0, diff --git a/federatedscope/autotune/run.py b/federatedscope/autotune/run.py index bdb2d27b6..0a453f416 100644 --- a/federatedscope/autotune/run.py +++ b/federatedscope/autotune/run.py @@ -8,7 +8,8 @@ def run_scheduler(scheduler, cfg, client_cfgs=None): cfg: The configurations of the FL course. client_cfgs: The clients' configurations. """ - if cfg.hpo.scheduler in ['sha', 'wrap_sha']: + # TODO: Fix 'wrap_sha' + if cfg.hpo.scheduler in ['sha']: _ = scheduler.optimize() elif cfg.hpo.scheduler in [ 'rs', 'bo_kde', 'hb', 'bohb', 'wrap_rs', 'wrap_bo_kde', 'wrap_hb', diff --git a/federatedscope/autotune/smac.py b/federatedscope/autotune/smac.py index 067f53147..ec8653979 100644 --- a/federatedscope/autotune/smac.py +++ b/federatedscope/autotune/smac.py @@ -11,8 +11,10 @@ def run_smac(cfg, scheduler, client_cfgs=None): + config_space = scheduler._search_space init_configs = [] perfs = [] + config_id = 0 def optimization_function_wrapper(config): """ @@ -23,6 +25,8 @@ def optimization_function_wrapper(config): Returns: Best results of server of specific FS run. """ + global config_id, config_space + budget = cfg.hpo.sha.budgets[-1] results = eval_in_fs(cfg, config, budget, client_cfgs) key1, key2 = cfg.hpo.metric.split('.') @@ -31,6 +35,7 @@ def optimization_function_wrapper(config): config['federate.total_round_num'] = budget init_configs.append(config) perfs.append(res) + config_id += 1 logger.info(f'Evaluate the {len(perfs)-1}-th config ' f'{config}, and get performance {res}') if cfg.wandb.use: @@ -51,12 +56,6 @@ def summarize(): return perfs - config_space = scheduler._search_space - if cfg.hpo.scheduler.startswith('wrap_'): - ss = CS.ConfigurationSpace() - ss.add_hyperparameter(config_space['hpo.table.idx']) - config_space = ss - if cfg.hpo.sha.iter != 0: n_iterations = cfg.hpo.sha.iter else: diff --git a/federatedscope/autotune/utils.py b/federatedscope/autotune/utils.py index ff7fe8bc3..0478c5900 100644 --- a/federatedscope/autotune/utils.py +++ b/federatedscope/autotune/utils.py @@ -1,6 +1,9 @@ +import os.path + import yaml import logging import pandas as pd +import numpy as np import ConfigSpace as CS from federatedscope.core.configs.yacs_config import CfgNode @@ -239,13 +242,15 @@ def process(file_path): plt.close() -def eval_in_fs(cfg, config, budget, client_cfgs=None): +def eval_in_fs(cfg, config, budget, config_id, ss, client_cfgs=None): """ Args: cfg: fs cfg config: sampled trial CS.Configuration budget: budget round for this trial + config_id: Identifier to generate somadditional files + ss: search space of HPO client_cfgs: client-wise cfg Returns: @@ -261,7 +266,17 @@ def eval_in_fs(cfg, config, budget, client_cfgs=None): if isinstance(config, CS.Configuration): config = dict(config) - # Add FedEx related keys to config + if 'wrap' in cfg.hpo.scheduler: + logger.info('scheduler wrapped by FedEx.') + config['hpo.fedex.ss'] = osp( + cfg.hpo.working_folder, f"{config_id}_tmp_grid_search_space.yaml") + if not os.path.exists(config['hpo.fedex.ss']): + generate_arm(cfg, config, config_id, ss) + + config['federate.save_to'] = osp(cfg.hpo.working_folder, + f"idx_{config_id}.pth") + config['federate.restore_from'] = osp(cfg.hpo.working_folder, + f"idx_{config_id}.pth") if 'hpo.table.idx' in config.keys(): idx = config['hpo.table.idx'] config['hpo.fedex.ss'] = osp(cfg.hpo.working_folder, @@ -300,6 +315,59 @@ def eval_in_fs(cfg, config, budget, client_cfgs=None): return results +def generate_arm(cfg, config, config_id, ss): + def make_local_perturbation(config): + neighbor = dict() + for k in config: + if 'fedex' in k or 'fedopt' in k or k in [ + 'federate.save_to', 'federate.total_round_num', 'eval.freq' + ]: + # a workaround + continue + hyper = ss.get(k) + if isinstance(hyper, CS.UniformFloatHyperparameter): + lb, ub = hyper.lower, hyper.upper + diameter = cfg.fedex.wrapper.eps * (ub - lb) + new_val = (config[k] - + 0.5 * diameter) + np.random.uniform() * diameter + neighbor[k] = float(np.clip(new_val, lb, ub)) + elif isinstance(hyper, CS.UniformIntegerHyperparameter): + lb, ub = hyper.lower, hyper.upper + diameter = cfg.fedex.wrapper.eps * (ub - lb) + new_val = round( + float((config[k] - 0.5 * diameter) + + np.random.uniform() * diameter)) + neighbor[k] = int(np.clip(new_val, lb, ub)) + elif isinstance(hyper, CS.CategoricalHyperparameter): + if len(hyper.choices) == 1: + neighbor[k] = config[k] + else: + threshold = cfg.fedex.wrapper.eps * len( + hyper.choices) / (len(hyper.choices) - 1) + rn = np.random.uniform() + new_val = np.random.choice( + hyper.choices) if rn <= threshold else config[k] + if type(new_val) in [np.int32, np.int64]: + neighbor[k] = int(new_val) + elif type(new_val) in [np.float32, np.float64]: + neighbor[k] = float(new_val) + else: + neighbor[k] = str(new_val) + else: + raise TypeError("Value of {} has an invalid type {}".format( + k, type(config[k]))) + + return neighbor + + arms = dict(("arm{}".format(1 + j), make_local_perturbation(config)) + for j in range(cfg.hpo.fedex.wrapper.arm - 1)) + arms['arm0'] = dict((k, v) for k, v in config.items() if k in arms['arm1']) + with open( + os.path.join(cfg.hpo.working_folder, + f'{config_id}_tmp_grid_search_space.yaml'), 'w') as f: + yaml.dump(arms, f) + + def flatten2nestdict(raw_dict, delimiter='.'): # TODO: delete this for the function of `arm2dict` def nested_set(dic, keys, value): diff --git a/federatedscope/core/configs/README.md b/federatedscope/core/configs/README.md index 98b809268..342aa97f0 100644 --- a/federatedscope/core/configs/README.md +++ b/federatedscope/core/configs/README.md @@ -360,11 +360,10 @@ These arguments are exposed for customizing our provided auto-tuning components. #### Wrappers for FedEx -| Name | (Type) Default Value | Description | Note | -|:----:|:--------------------:|:-------------------------------------------|:-----| -| `hpo.table.eps` | (float) 0.1 | The probability to make local perturbation. | Larger values lead to drastically different arms of the bandit FedEx attempts to solve. | -| `hpo.table.num` | (int) 27 | The number of arms of the bandit FedEx attempts to solve. | - | -| `hpo.table.idx` | (int) 0 | The key (i.e., name) of the hyperparameter wrapper considers. | No need to change this argument. | +| Name | (Type) Default Value | Description | Note | +|:-----------------------:|:--------------------:|:-------------------------------------------|:-----| +| `hpo.fedex.wrapper.eps` | (float) 0.1 | The probability to make local perturbation. | Larger values lead to drastically different arms of the bandit FedEx attempts to solve. | +| `hpo.fedex.wrapper.arm` | (int) 27 | The number of arms of the bandit FedEx attempts to solve. | - | ### Attack diff --git a/federatedscope/core/configs/cfg_hpo.py b/federatedscope/core/configs/cfg_hpo.py index b48b7105c..f6f41e55f 100644 --- a/federatedscope/core/configs/cfg_hpo.py +++ b/federatedscope/core/configs/cfg_hpo.py @@ -46,16 +46,15 @@ def extend_hpo_cfg(cfg): cfg.hpo.fedex.psn = False cfg.hpo.fedex.pi_lr = 0.01 + # FedEx wrap optimizer related args + cfg.hpo.fedex.wrapper = CN() + cfg.hpo.fedex.wrapper.eps = 0.1 + cfg.hpo.fedex.wrapper.arm = 27 + cfg.hpo.fedex.attack = CN() cfg.hpo.fedex.attack.id = [] # client IDs who inject noise into policy cfg.hpo.fedex.attack.sigma = 1.0 # sigma of white noise - # Table - cfg.hpo.table = CN() - cfg.hpo.table.eps = 0.1 - cfg.hpo.table.num = 27 - cfg.hpo.table.idx = 0 - def assert_hpo_cfg(cfg): # HPO related From 5a16d57f4638d125606a0bdad5434fcda5331215 Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Fri, 13 Jan 2023 16:40:10 +0800 Subject: [PATCH 23/38] update interface --- scripts/hpo_exp_scripts/usability/pubmed/bo_gp.yaml | 3 --- scripts/hpo_exp_scripts/usability/pubmed/bo_gp_wrap.yaml | 3 --- scripts/hpo_exp_scripts/usability/pubmed/bo_kde.yaml | 3 --- scripts/hpo_exp_scripts/usability/pubmed/bo_kde_wrap.yaml | 3 --- scripts/hpo_exp_scripts/usability/pubmed/bo_rf.yaml | 3 --- scripts/hpo_exp_scripts/usability/pubmed/bo_rf_wrap.yaml | 3 --- scripts/hpo_exp_scripts/usability/pubmed/bohb.yaml | 3 --- scripts/hpo_exp_scripts/usability/pubmed/bohb_wrap.yaml | 3 --- scripts/hpo_exp_scripts/usability/pubmed/hb.yaml | 3 --- scripts/hpo_exp_scripts/usability/pubmed/hb_wrap.yaml | 3 --- scripts/hpo_exp_scripts/usability/pubmed/rs.yaml | 3 --- scripts/hpo_exp_scripts/usability/pubmed/rs_wrap.yaml | 3 --- 12 files changed, 36 deletions(-) diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bo_gp.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bo_gp.yaml index 44501956f..6bcebf3b1 100644 --- a/scripts/hpo_exp_scripts/usability/pubmed/bo_gp.yaml +++ b/scripts/hpo_exp_scripts/usability/pubmed/bo_gp.yaml @@ -41,13 +41,10 @@ dataloader: hpo: scheduler: bo_gp num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' sha: budgets: [ 50, 50 ] elim_rate: 3 iter: 50 - table: - num: 27 metric: 'server_global_eval.val_avg_loss' working_folder: bo_gp_pubmed_avg \ No newline at end of file diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bo_gp_wrap.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bo_gp_wrap.yaml index 61467c046..43d1e449e 100644 --- a/scripts/hpo_exp_scripts/usability/pubmed/bo_gp_wrap.yaml +++ b/scripts/hpo_exp_scripts/usability/pubmed/bo_gp_wrap.yaml @@ -42,14 +42,11 @@ dataloader: hpo: scheduler: wrap_bo_gp num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' sha: budgets: [ 50, 50 ] elim_rate: 3 iter: 50 - table: - num: 27 fedex: sched: 'aggressive' use: True diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bo_kde.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bo_kde.yaml index fef15d9a3..7e185cd6f 100644 --- a/scripts/hpo_exp_scripts/usability/pubmed/bo_kde.yaml +++ b/scripts/hpo_exp_scripts/usability/pubmed/bo_kde.yaml @@ -41,13 +41,10 @@ dataloader: hpo: scheduler: bo_kde num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' sha: budgets: [ 50, 50 ] elim_rate: 3 iter: 50 - table: - num: 27 metric: 'server_global_eval.val_avg_loss' working_folder: bo_kde_pubmed_avg \ No newline at end of file diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bo_kde_wrap.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bo_kde_wrap.yaml index de3a45ad5..b7e58f604 100644 --- a/scripts/hpo_exp_scripts/usability/pubmed/bo_kde_wrap.yaml +++ b/scripts/hpo_exp_scripts/usability/pubmed/bo_kde_wrap.yaml @@ -42,14 +42,11 @@ dataloader: hpo: scheduler: wrap_bo_kde num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' sha: budgets: [ 50, 50 ] elim_rate: 3 iter: 50 - table: - num: 27 fedex: sched: 'aggressive' use: True diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bo_rf.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bo_rf.yaml index c07591a71..a8ff27a9e 100644 --- a/scripts/hpo_exp_scripts/usability/pubmed/bo_rf.yaml +++ b/scripts/hpo_exp_scripts/usability/pubmed/bo_rf.yaml @@ -41,13 +41,10 @@ dataloader: hpo: scheduler: bo_rf num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' sha: budgets: [ 50, 50 ] elim_rate: 3 iter: 50 - table: - num: 27 metric: 'server_global_eval.val_avg_loss' working_folder: bo_rf_pubmed_avg \ No newline at end of file diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bo_rf_wrap.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bo_rf_wrap.yaml index bd130b773..fbdb20efe 100644 --- a/scripts/hpo_exp_scripts/usability/pubmed/bo_rf_wrap.yaml +++ b/scripts/hpo_exp_scripts/usability/pubmed/bo_rf_wrap.yaml @@ -42,14 +42,11 @@ dataloader: hpo: scheduler: wrap_bo_rf num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' sha: budgets: [ 50, 50 ] elim_rate: 3 iter: 50 - table: - num: 27 fedex: sched: 'aggressive' use: True diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bohb.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bohb.yaml index 858ebc55a..d80b62efa 100644 --- a/scripts/hpo_exp_scripts/usability/pubmed/bohb.yaml +++ b/scripts/hpo_exp_scripts/usability/pubmed/bohb.yaml @@ -41,13 +41,10 @@ dataloader: hpo: scheduler: bohb num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' sha: budgets: [ 9, 81 ] elim_rate: 3 iter: 12 - table: - num: 27 metric: 'server_global_eval.val_avg_loss' working_folder: bohb_pubmed_avg \ No newline at end of file diff --git a/scripts/hpo_exp_scripts/usability/pubmed/bohb_wrap.yaml b/scripts/hpo_exp_scripts/usability/pubmed/bohb_wrap.yaml index 2fde86467..752f1f0ea 100644 --- a/scripts/hpo_exp_scripts/usability/pubmed/bohb_wrap.yaml +++ b/scripts/hpo_exp_scripts/usability/pubmed/bohb_wrap.yaml @@ -42,14 +42,11 @@ dataloader: hpo: scheduler: wrap_bohb num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' sha: budgets: [ 9, 81 ] elim_rate: 3 iter: 12 - table: - num: 27 fedex: sched: 'aggressive' use: True diff --git a/scripts/hpo_exp_scripts/usability/pubmed/hb.yaml b/scripts/hpo_exp_scripts/usability/pubmed/hb.yaml index 82b634bb0..d0aace870 100644 --- a/scripts/hpo_exp_scripts/usability/pubmed/hb.yaml +++ b/scripts/hpo_exp_scripts/usability/pubmed/hb.yaml @@ -41,13 +41,10 @@ dataloader: hpo: scheduler: hb num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' sha: budgets: [ 9, 81 ] elim_rate: 3 iter: 12 - table: - num: 27 metric: 'server_global_eval.val_avg_loss' working_folder: hb_pubmed_avg \ No newline at end of file diff --git a/scripts/hpo_exp_scripts/usability/pubmed/hb_wrap.yaml b/scripts/hpo_exp_scripts/usability/pubmed/hb_wrap.yaml index d5d2e2ecf..69b5e5683 100644 --- a/scripts/hpo_exp_scripts/usability/pubmed/hb_wrap.yaml +++ b/scripts/hpo_exp_scripts/usability/pubmed/hb_wrap.yaml @@ -42,14 +42,11 @@ dataloader: hpo: scheduler: wrap_hb num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' sha: budgets: [ 9, 81 ] elim_rate: 3 iter: 12 - table: - num: 27 fedex: sched: 'aggressive' use: True diff --git a/scripts/hpo_exp_scripts/usability/pubmed/rs.yaml b/scripts/hpo_exp_scripts/usability/pubmed/rs.yaml index 4aa6032c1..b748f26ce 100644 --- a/scripts/hpo_exp_scripts/usability/pubmed/rs.yaml +++ b/scripts/hpo_exp_scripts/usability/pubmed/rs.yaml @@ -41,13 +41,10 @@ dataloader: hpo: scheduler: rs num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' sha: budgets: [ 50, 50 ] elim_rate: 3 iter: 50 - table: - num: 27 metric: 'server_global_eval.val_avg_loss' working_folder: rs_pubmed_avg \ No newline at end of file diff --git a/scripts/hpo_exp_scripts/usability/pubmed/rs_wrap.yaml b/scripts/hpo_exp_scripts/usability/pubmed/rs_wrap.yaml index 75b8c89db..d02dc104f 100644 --- a/scripts/hpo_exp_scripts/usability/pubmed/rs_wrap.yaml +++ b/scripts/hpo_exp_scripts/usability/pubmed/rs_wrap.yaml @@ -42,14 +42,11 @@ dataloader: hpo: scheduler: wrap_rs num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/usability/pubmed/ss.yaml' sha: budgets: [ 50, 50 ] elim_rate: 3 iter: 50 - table: - num: 27 fedex: sched: 'aggressive' use: True From a08cef9e0dd5d1c0dbc7cfee6dbfd226dcd059af Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Fri, 13 Jan 2023 16:50:25 +0800 Subject: [PATCH 24/38] fix minor bugs --- federatedscope/autotune/smac.py | 9 +++++---- federatedscope/autotune/utils.py | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/federatedscope/autotune/smac.py b/federatedscope/autotune/smac.py index ec8653979..8f4e60294 100644 --- a/federatedscope/autotune/smac.py +++ b/federatedscope/autotune/smac.py @@ -1,6 +1,5 @@ import logging import numpy as np -import ConfigSpace as CS from federatedscope.autotune.utils import eval_in_fs, log2wandb from smac.facade.smac_bb_facade import SMAC4BB from smac.facade.smac_hpo_facade import SMAC4HPO @@ -9,12 +8,13 @@ logging.basicConfig(level=logging.WARNING) logger = logging.getLogger(__name__) +config_id = 0 + def run_smac(cfg, scheduler, client_cfgs=None): config_space = scheduler._search_space init_configs = [] perfs = [] - config_id = 0 def optimization_function_wrapper(config): """ @@ -25,10 +25,11 @@ def optimization_function_wrapper(config): Returns: Best results of server of specific FS run. """ - global config_id, config_space + global config_id budget = cfg.hpo.sha.budgets[-1] - results = eval_in_fs(cfg, config, budget, client_cfgs) + results = eval_in_fs(cfg, config, budget, config_id, config_space, + client_cfgs) key1, key2 = cfg.hpo.metric.split('.') res = results[key1][key2] config = dict(config) diff --git a/federatedscope/autotune/utils.py b/federatedscope/autotune/utils.py index 0478c5900..d4bb693ab 100644 --- a/federatedscope/autotune/utils.py +++ b/federatedscope/autotune/utils.py @@ -327,13 +327,13 @@ def make_local_perturbation(config): hyper = ss.get(k) if isinstance(hyper, CS.UniformFloatHyperparameter): lb, ub = hyper.lower, hyper.upper - diameter = cfg.fedex.wrapper.eps * (ub - lb) + diameter = cfg.hpo.fedex.wrapper.eps * (ub - lb) new_val = (config[k] - 0.5 * diameter) + np.random.uniform() * diameter neighbor[k] = float(np.clip(new_val, lb, ub)) elif isinstance(hyper, CS.UniformIntegerHyperparameter): lb, ub = hyper.lower, hyper.upper - diameter = cfg.fedex.wrapper.eps * (ub - lb) + diameter = cfg.hpo.fedex.wrapper.eps * (ub - lb) new_val = round( float((config[k] - 0.5 * diameter) + np.random.uniform() * diameter)) @@ -342,7 +342,7 @@ def make_local_perturbation(config): if len(hyper.choices) == 1: neighbor[k] = config[k] else: - threshold = cfg.fedex.wrapper.eps * len( + threshold = cfg.hpo.fedex.wrapper.eps * len( hyper.choices) / (len(hyper.choices) - 1) rn = np.random.uniform() new_val = np.random.choice( From 0c674276bedbbda0c020bd60683a09334c4e892d Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Fri, 13 Jan 2023 18:03:16 +0800 Subject: [PATCH 25/38] add config_id --- federatedscope/autotune/hpbandster.py | 5 +++-- federatedscope/autotune/smac.py | 3 ++- federatedscope/autotune/utils.py | 18 +++++++++--------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/federatedscope/autotune/hpbandster.py b/federatedscope/autotune/hpbandster.py index 26415b507..e44798ab4 100644 --- a/federatedscope/autotune/hpbandster.py +++ b/federatedscope/autotune/hpbandster.py @@ -83,7 +83,7 @@ def compute(self, config, budget, config_id, **kwargs): log2wandb(len(self._perfs) - 1, config, results, self.cfg) return {'loss': float(res), 'info': res} - def summarize(self): + def summarize(self, res=None): from federatedscope.autotune.utils import summarize_hpo_results results = summarize_hpo_results(self._init_configs, self._perfs, @@ -94,6 +94,7 @@ def summarize(self): "========================== HPO Final ==========================") logger.info("\n{}".format(results)) logger.info("====================================================") + logger.info(f'Winner config_id: {res.get_incumbent_id()}') return results @@ -139,6 +140,6 @@ def run_hpbandster(cfg, scheduler, client_cfgs=None): optimizer.shutdown(shutdown_workers=True) NS.shutdown() all_runs = res.get_all_runs() - w.summarize() + w.summarize(res) return [x.info for x in all_runs] diff --git a/federatedscope/autotune/smac.py b/federatedscope/autotune/smac.py index 8f4e60294..0092728e8 100644 --- a/federatedscope/autotune/smac.py +++ b/federatedscope/autotune/smac.py @@ -54,8 +54,9 @@ def summarize(): "========================== HPO Final ==========================") logger.info("\n{}".format(results)) logger.info("====================================================") + logger.info(f'Winner config_id: {np.argmin(perfs)}') - return perfs + return results if cfg.hpo.sha.iter != 0: n_iterations = cfg.hpo.sha.iter diff --git a/federatedscope/autotune/utils.py b/federatedscope/autotune/utils.py index d4bb693ab..6dc4b8edb 100644 --- a/federatedscope/autotune/utils.py +++ b/federatedscope/autotune/utils.py @@ -267,7 +267,7 @@ def eval_in_fs(cfg, config, budget, config_id, ss, client_cfgs=None): if isinstance(config, CS.Configuration): config = dict(config) if 'wrap' in cfg.hpo.scheduler: - logger.info('scheduler wrapped by FedEx.') + logger.info('FedEx is wrapped by scheduler.') config['hpo.fedex.ss'] = osp( cfg.hpo.working_folder, f"{config_id}_tmp_grid_search_space.yaml") if not os.path.exists(config['hpo.fedex.ss']): @@ -277,14 +277,14 @@ def eval_in_fs(cfg, config, budget, config_id, ss, client_cfgs=None): f"idx_{config_id}.pth") config['federate.restore_from'] = osp(cfg.hpo.working_folder, f"idx_{config_id}.pth") - if 'hpo.table.idx' in config.keys(): - idx = config['hpo.table.idx'] - config['hpo.fedex.ss'] = osp(cfg.hpo.working_folder, - f"{idx}_tmp_grid_search_space.yaml") - config['federate.save_to'] = osp(cfg.hpo.working_folder, - f"idx_{idx}.pth") - config['federate.restore_from'] = osp(cfg.hpo.working_folder, - f"idx_{idx}.pth") + # if 'hpo.table.idx' in config.keys(): + # idx = config['hpo.table.idx'] + # config['hpo.fedex.ss'] = osp(cfg.hpo.working_folder, + # f"{idx}_tmp_grid_search_space.yaml") + # config['federate.save_to'] = osp(cfg.hpo.working_folder, + # f"idx_{idx}.pth") + # config['federate.restore_from'] = osp(cfg.hpo.working_folder, + # f"idx_{idx}.pth") # Global cfg trial_cfg = cfg.clone() # specify the configuration of interest From 2db5c649fb73789fc9f6693b0f72c4886226086d Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Fri, 13 Jan 2023 18:25:03 +0800 Subject: [PATCH 26/38] add info to show --- federatedscope/autotune/utils.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/federatedscope/autotune/utils.py b/federatedscope/autotune/utils.py index 6dc4b8edb..910d77efc 100644 --- a/federatedscope/autotune/utils.py +++ b/federatedscope/autotune/utils.py @@ -277,14 +277,6 @@ def eval_in_fs(cfg, config, budget, config_id, ss, client_cfgs=None): f"idx_{config_id}.pth") config['federate.restore_from'] = osp(cfg.hpo.working_folder, f"idx_{config_id}.pth") - # if 'hpo.table.idx' in config.keys(): - # idx = config['hpo.table.idx'] - # config['hpo.fedex.ss'] = osp(cfg.hpo.working_folder, - # f"{idx}_tmp_grid_search_space.yaml") - # config['federate.save_to'] = osp(cfg.hpo.working_folder, - # f"idx_{idx}.pth") - # config['federate.restore_from'] = osp(cfg.hpo.working_folder, - # f"idx_{idx}.pth") # Global cfg trial_cfg = cfg.clone() # specify the configuration of interest From 3cc47b7dbd4c31c99507d536769ffc32e5e8fa62 Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Mon, 16 Jan 2023 14:27:36 +0800 Subject: [PATCH 27/38] fix yaml for hpo --- scripts/example_configs/femnist/avg/bo_gp_wrap.yaml | 7 ++----- scripts/example_configs/femnist/avg/bo_rf_wrap.yaml | 9 +++------ scripts/example_configs/femnist/avg/bohb_wrap.yaml | 8 +++----- scripts/example_configs/femnist/avg/hb_wrap.yaml | 4 ++-- scripts/example_configs/femnist/avg/rs_wrap.yaml | 4 ++-- scripts/hpo_exp_scripts/parse_exp_results.py | 3 +++ .../hpo_exp_scripts/robustness/attack/bo_gp_wrap.yaml | 3 --- .../hpo_exp_scripts/robustness/attack/bo_kde_wrap.yaml | 3 --- .../hpo_exp_scripts/robustness/attack/bo_rf_wrap.yaml | 3 --- scripts/hpo_exp_scripts/robustness/attack/bohb_wrap.yaml | 3 --- scripts/hpo_exp_scripts/robustness/attack/hb_wrap.yaml | 3 --- scripts/hpo_exp_scripts/robustness/attack/rs_wrap.yaml | 3 --- scripts/hpo_exp_scripts/robustness/attack/run.sh | 3 ++- .../hpo_exp_scripts/uniqueness/cifar10/bo_gp_wrap.yaml | 3 --- .../hpo_exp_scripts/uniqueness/cifar10/bo_kde_wrap.yaml | 3 --- .../hpo_exp_scripts/uniqueness/cifar10/bo_rf_wrap.yaml | 3 --- .../hpo_exp_scripts/uniqueness/cifar10/bohb_wrap.yaml | 3 --- scripts/hpo_exp_scripts/uniqueness/cifar10/hb_wrap.yaml | 3 --- scripts/hpo_exp_scripts/uniqueness/cifar10/rs_wrap.yaml | 3 --- scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh | 3 ++- .../uniqueness/personalized/bo_gp_wrap.yaml | 3 --- .../uniqueness/personalized/bo_kde_wrap.yaml | 2 -- .../uniqueness/personalized/bo_rf_wrap.yaml | 2 -- .../uniqueness/personalized/bohb_wrap.yaml | 3 --- .../hpo_exp_scripts/uniqueness/personalized/hb_wrap.yaml | 3 --- .../hpo_exp_scripts/uniqueness/personalized/rs_wrap.yaml | 3 --- scripts/hpo_exp_scripts/uniqueness/personalized/run.sh | 3 ++- scripts/hpo_exp_scripts/usability/pubmed/run.sh | 3 ++- 28 files changed, 23 insertions(+), 76 deletions(-) create mode 100644 scripts/hpo_exp_scripts/parse_exp_results.py diff --git a/scripts/example_configs/femnist/avg/bo_gp_wrap.yaml b/scripts/example_configs/femnist/avg/bo_gp_wrap.yaml index 815325682..f9bca6368 100644 --- a/scripts/example_configs/femnist/avg/bo_gp_wrap.yaml +++ b/scripts/example_configs/femnist/avg/bo_gp_wrap.yaml @@ -7,7 +7,7 @@ seed: 12345 federate: mode: standalone total_round_num: 500 - sample_client_rate: 1.0 + sample_client_rate: 0.01 share_local_model: True online_aggr: True use_diff: True @@ -43,14 +43,11 @@ eval: hpo: scheduler: wrap_bo_gp num_workers: 0 - init_cand_num: 100 ss: 'scripts/example_configs/femnist/avg/ss.yaml' sha: - budgets: [ 50, 50 ] + budgets: [ 3, 3 ] elim_rate: 3 iter: 50 - table: - num: 27 fedex: sched: 'aggressive' use: True diff --git a/scripts/example_configs/femnist/avg/bo_rf_wrap.yaml b/scripts/example_configs/femnist/avg/bo_rf_wrap.yaml index f2977a2eb..2756f178b 100644 --- a/scripts/example_configs/femnist/avg/bo_rf_wrap.yaml +++ b/scripts/example_configs/femnist/avg/bo_rf_wrap.yaml @@ -7,7 +7,7 @@ seed: 12345 federate: mode: standalone total_round_num: 500 - sample_client_rate: 1.0 + sample_client_rate: 0.01 share_local_model: True online_aggr: True use_diff: True @@ -43,14 +43,11 @@ eval: hpo: scheduler: wrap_bo_rf num_workers: 0 - init_cand_num: 100 ss: 'scripts/example_configs/femnist/avg/ss.yaml' sha: - budgets: [ 50, 50 ] + budgets: [ 3, 3 ] elim_rate: 3 - iter: 50 - table: - num: 27 + iter: 10 fedex: sched: 'aggressive' use: True diff --git a/scripts/example_configs/femnist/avg/bohb_wrap.yaml b/scripts/example_configs/femnist/avg/bohb_wrap.yaml index e3ca1f733..76c1e6d5f 100644 --- a/scripts/example_configs/femnist/avg/bohb_wrap.yaml +++ b/scripts/example_configs/femnist/avg/bohb_wrap.yaml @@ -7,7 +7,7 @@ seed: 12345 federate: mode: standalone total_round_num: 500 - sample_client_rate: 1.0 + sample_client_rate: 0.01 share_local_model: True online_aggr: True use_diff: True @@ -46,11 +46,9 @@ hpo: init_cand_num: 100 ss: 'scripts/example_configs/femnist/avg/ss.yaml' sha: - budgets: [ 9, 81 ] + budgets: [ 1, 9 ] elim_rate: 3 - iter: 12 - table: - num: 27 + iter: 6 fedex: sched: 'aggressive' use: True diff --git a/scripts/example_configs/femnist/avg/hb_wrap.yaml b/scripts/example_configs/femnist/avg/hb_wrap.yaml index 1cfdeeca6..64fc90889 100644 --- a/scripts/example_configs/femnist/avg/hb_wrap.yaml +++ b/scripts/example_configs/femnist/avg/hb_wrap.yaml @@ -7,7 +7,7 @@ seed: 12345 federate: mode: standalone total_round_num: 500 - sample_client_rate: 1.0 + sample_client_rate: 0.01 share_local_model: True online_aggr: True use_diff: True @@ -46,7 +46,7 @@ hpo: init_cand_num: 100 ss: 'scripts/example_configs/femnist/avg/ss.yaml' sha: - budgets: [ 9, 81 ] + budgets: [ 1, 9 ] elim_rate: 3 iter: 12 table: diff --git a/scripts/example_configs/femnist/avg/rs_wrap.yaml b/scripts/example_configs/femnist/avg/rs_wrap.yaml index 9d4680524..00f45d58e 100644 --- a/scripts/example_configs/femnist/avg/rs_wrap.yaml +++ b/scripts/example_configs/femnist/avg/rs_wrap.yaml @@ -49,13 +49,13 @@ hpo: budgets: [ 50, 50 ] elim_rate: 3 iter: 50 - table: - num: 27 fedex: sched: 'aggressive' use: True diff: False eta0: -1.0 gamma: 0.0 + wrapper: + arm: 27 metric: 'client_summarized_weighted_avg.val_avg_loss' working_folder: rs_wrap_femnist_avg diff --git a/scripts/hpo_exp_scripts/parse_exp_results.py b/scripts/hpo_exp_scripts/parse_exp_results.py new file mode 100644 index 000000000..5d252c689 --- /dev/null +++ b/scripts/hpo_exp_scripts/parse_exp_results.py @@ -0,0 +1,3 @@ +# TODO: implement this +def parse(): + ... diff --git a/scripts/hpo_exp_scripts/robustness/attack/bo_gp_wrap.yaml b/scripts/hpo_exp_scripts/robustness/attack/bo_gp_wrap.yaml index 92a06cbe2..e5aaa4afd 100644 --- a/scripts/hpo_exp_scripts/robustness/attack/bo_gp_wrap.yaml +++ b/scripts/hpo_exp_scripts/robustness/attack/bo_gp_wrap.yaml @@ -43,14 +43,11 @@ eval: hpo: scheduler: wrap_bo_gp num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/robustness/attack/ss.yaml' sha: budgets: [ 50, 50 ] elim_rate: 3 iter: 50 - table: - num: 27 fedex: sched: 'aggressive' use: True diff --git a/scripts/hpo_exp_scripts/robustness/attack/bo_kde_wrap.yaml b/scripts/hpo_exp_scripts/robustness/attack/bo_kde_wrap.yaml index c66642752..a365f7956 100644 --- a/scripts/hpo_exp_scripts/robustness/attack/bo_kde_wrap.yaml +++ b/scripts/hpo_exp_scripts/robustness/attack/bo_kde_wrap.yaml @@ -43,14 +43,11 @@ eval: hpo: scheduler: wrap_bo_kde num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/robustness/attack/ss.yaml' sha: budgets: [ 50, 50 ] elim_rate: 3 iter: 50 - table: - num: 27 fedex: sched: 'aggressive' use: True diff --git a/scripts/hpo_exp_scripts/robustness/attack/bo_rf_wrap.yaml b/scripts/hpo_exp_scripts/robustness/attack/bo_rf_wrap.yaml index 906775e5e..81f29aa2e 100644 --- a/scripts/hpo_exp_scripts/robustness/attack/bo_rf_wrap.yaml +++ b/scripts/hpo_exp_scripts/robustness/attack/bo_rf_wrap.yaml @@ -43,14 +43,11 @@ eval: hpo: scheduler: wrap_bo_rf num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/robustness/attack/ss.yaml' sha: budgets: [ 50, 50 ] elim_rate: 3 iter: 50 - table: - num: 27 fedex: sched: 'aggressive' use: True diff --git a/scripts/hpo_exp_scripts/robustness/attack/bohb_wrap.yaml b/scripts/hpo_exp_scripts/robustness/attack/bohb_wrap.yaml index 720356da7..5bfb9c834 100644 --- a/scripts/hpo_exp_scripts/robustness/attack/bohb_wrap.yaml +++ b/scripts/hpo_exp_scripts/robustness/attack/bohb_wrap.yaml @@ -43,14 +43,11 @@ eval: hpo: scheduler: wrap_bohb num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/robustness/attack/ss.yaml' sha: budgets: [ 9, 81 ] elim_rate: 3 iter: 12 - table: - num: 27 fedex: sched: 'aggressive' use: True diff --git a/scripts/hpo_exp_scripts/robustness/attack/hb_wrap.yaml b/scripts/hpo_exp_scripts/robustness/attack/hb_wrap.yaml index b91fa7f15..56794c4f0 100644 --- a/scripts/hpo_exp_scripts/robustness/attack/hb_wrap.yaml +++ b/scripts/hpo_exp_scripts/robustness/attack/hb_wrap.yaml @@ -43,14 +43,11 @@ eval: hpo: scheduler: wrap_hb num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/robustness/attack/ss.yaml' sha: budgets: [ 9, 81 ] elim_rate: 3 iter: 12 - table: - num: 27 fedex: sched: 'aggressive' use: True diff --git a/scripts/hpo_exp_scripts/robustness/attack/rs_wrap.yaml b/scripts/hpo_exp_scripts/robustness/attack/rs_wrap.yaml index 8a332bcf4..06ea2b67e 100644 --- a/scripts/hpo_exp_scripts/robustness/attack/rs_wrap.yaml +++ b/scripts/hpo_exp_scripts/robustness/attack/rs_wrap.yaml @@ -43,14 +43,11 @@ eval: hpo: scheduler: wrap_rs num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/robustness/attack/ss.yaml' sha: budgets: [ 50, 50 ] elim_rate: 3 iter: 50 - table: - num: 27 fedex: sched: 'aggressive' use: True diff --git a/scripts/hpo_exp_scripts/robustness/attack/run.sh b/scripts/hpo_exp_scripts/robustness/attack/run.sh index 1ab004ef1..128d0371e 100644 --- a/scripts/hpo_exp_scripts/robustness/attack/run.sh +++ b/scripts/hpo_exp_scripts/robustness/attack/run.sh @@ -3,11 +3,12 @@ set -e attacker=$1 sigma=$2 seed=$3 +device=$4 methods=('rs_wrap' 'bo_gp_wrap' 'bo_kde_wrap' 'bo_rf_wrap' 'hb_wrap' 'bohb_wrap') for (( m=0; m<${#methods[@]}; m++ )) do - nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/robustness/attack/${methods[$m]}.yaml device $((m%8)) seed ${seed} hpo.fedex.attack.id ${attacker} hpo.fedex.attack.sigma ${sigma} >/dev/null 2>&1 & + nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/robustness/attack/${methods[$m]}.yaml device $((m%${device})) seed ${seed} hpo.fedex.attack.id ${attacker} hpo.fedex.attack.sigma ${sigma} >/dev/null 2>&1 & done diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_gp_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_gp_wrap.yaml index b98e2f488..867d0424f 100644 --- a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_gp_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_gp_wrap.yaml @@ -48,14 +48,11 @@ eval: hpo: scheduler: wrap_bo_gp num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' sha: budgets: [ 50, 50 ] elim_rate: 3 iter: 50 - table: - num: 27 fedex: sched: 'aggressive' use: True diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_kde_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_kde_wrap.yaml index 588778c17..5c57fa254 100644 --- a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_kde_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_kde_wrap.yaml @@ -48,14 +48,11 @@ eval: hpo: scheduler: wrap_bo_kde num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' sha: budgets: [ 50, 50 ] elim_rate: 3 iter: 50 - table: - num: 27 fedex: sched: 'aggressive' use: True diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_rf_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_rf_wrap.yaml index f2b54e306..83808b669 100644 --- a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_rf_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_rf_wrap.yaml @@ -48,14 +48,11 @@ eval: hpo: scheduler: wrap_bo_rf num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' sha: budgets: [ 50, 50 ] elim_rate: 3 iter: 50 - table: - num: 27 fedex: sched: 'aggressive' use: True diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bohb_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bohb_wrap.yaml index e7bd45b97..df3c80eb4 100644 --- a/scripts/hpo_exp_scripts/uniqueness/cifar10/bohb_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bohb_wrap.yaml @@ -48,14 +48,11 @@ eval: hpo: scheduler: wrap_bohb num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' sha: budgets: [ 9, 81 ] elim_rate: 3 iter: 12 - table: - num: 27 fedex: sched: 'aggressive' use: True diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/hb_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/hb_wrap.yaml index 11242ce85..a1dfe0835 100644 --- a/scripts/hpo_exp_scripts/uniqueness/cifar10/hb_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/hb_wrap.yaml @@ -48,14 +48,11 @@ eval: hpo: scheduler: wrap_hb num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' sha: budgets: [ 9, 81 ] elim_rate: 3 iter: 12 - table: - num: 27 fedex: sched: 'aggressive' use: True diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/rs_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/rs_wrap.yaml index e9f9fcc81..e674f98ed 100644 --- a/scripts/hpo_exp_scripts/uniqueness/cifar10/rs_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/rs_wrap.yaml @@ -48,14 +48,11 @@ eval: hpo: scheduler: wrap_rs num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' sha: budgets: [ 50, 50 ] elim_rate: 3 iter: 50 - table: - num: 27 fedex: sched: 'aggressive' use: True diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh b/scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh index c5faaa4bf..5bfcca215 100644 --- a/scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh @@ -15,11 +15,12 @@ set -e alpha=$1 seed=$2 +device=$3 methods=('rs' 'rs_wrap' 'bo_gp' 'bo_gp_wrap' 'bo_kde' 'bo_kde_wrap' 'bo_rf' 'bo_rf_wrap' 'hb' 'hb_wrap' 'bohb' 'bohb_wrap') for (( m=0; m<${#methods[@]}; m++ )) do - nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/uniqueness/cifar10/${methods[$m]}.yaml device $((m%4)) seed ${seed} data.splitter_args "[{'alpha': ${alpha}}]" >/dev/null 2>&1 & + nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/uniqueness/cifar10/${methods[$m]}.yaml device $((m%${device})) seed ${seed} data.splitter_args "[{'alpha': ${alpha}}]" >/dev/null 2>&1 & done diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp_wrap.yaml index ec1d583a9..b447f72bb 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp_wrap.yaml @@ -43,14 +43,11 @@ eval: hpo: scheduler: wrap_bo_gp num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' sha: budgets: [ 50, 50 ] elim_rate: 3 iter: 50 - table: - num: 27 fedex: psn: True pi_lr: 0.01 diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde_wrap.yaml index d6e3e2c62..d90a5c5f8 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde_wrap.yaml @@ -49,8 +49,6 @@ hpo: budgets: [ 50, 50 ] elim_rate: 3 iter: 50 - table: - num: 27 fedex: psn: True pi_lr: 0.01 diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf_wrap.yaml index 165ee1a1f..a0abd0f25 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf_wrap.yaml @@ -49,8 +49,6 @@ hpo: budgets: [ 50, 50 ] elim_rate: 3 iter: 50 - table: - num: 27 fedex: psn: True pi_lr: 0.01 diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bohb_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bohb_wrap.yaml index 59a9089b3..d4d374b7a 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/bohb_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bohb_wrap.yaml @@ -43,14 +43,11 @@ eval: hpo: scheduler: wrap_bohb num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' sha: budgets: [ 9, 81 ] elim_rate: 3 iter: 12 - table: - num: 27 fedex: psn: True pi_lr: 0.01 diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/hb_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/hb_wrap.yaml index 94771f381..d7605598a 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/hb_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/hb_wrap.yaml @@ -43,14 +43,11 @@ eval: hpo: scheduler: wrap_hb num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' sha: budgets: [ 9, 81 ] elim_rate: 3 iter: 12 - table: - num: 27 fedex: psn: True pi_lr: 0.01 diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/rs_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/rs_wrap.yaml index 24860791a..1bdf14d4a 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/rs_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/rs_wrap.yaml @@ -43,14 +43,11 @@ eval: hpo: scheduler: wrap_rs num_workers: 0 - init_cand_num: 100 ss: 'scripts/hpo_exp_scripts/uniqueness/personalized/ss.yaml' sha: budgets: [ 50, 50 ] elim_rate: 3 iter: 50 - table: - num: 27 fedex: psn: True pi_lr: 0.01 diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/run.sh b/scripts/hpo_exp_scripts/uniqueness/personalized/run.sh index cf6bb4f7f..36e75e8da 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/run.sh +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/run.sh @@ -1,11 +1,12 @@ set -e seed=$1 +device=$2 methods=('rs' 'rs_wrap' 'bo_gp' 'bo_gp_wrap' 'bo_kde' 'bo_kde_wrap' 'bo_rf' 'bo_rf_wrap' 'hb' 'hb_wrap' 'bohb' 'bohb_wrap') for (( m=0; m<${#methods[@]}; m++ )) do - nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/uniqueness/personalized/${methods[$m]}.yaml device $((m%4)) seed ${seed} >/dev/null 2>&1 + nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/uniqueness/personalized/${methods[$m]}.yaml device $((m%${device})) seed ${seed} >/dev/null 2>&1 done \ No newline at end of file diff --git a/scripts/hpo_exp_scripts/usability/pubmed/run.sh b/scripts/hpo_exp_scripts/usability/pubmed/run.sh index 483b2d638..b4e3da1bd 100644 --- a/scripts/hpo_exp_scripts/usability/pubmed/run.sh +++ b/scripts/hpo_exp_scripts/usability/pubmed/run.sh @@ -1,11 +1,12 @@ set -e seed=$1 +device=$2 methods=('rs' 'rs_wrap' 'bo_gp' 'bo_gp_wrap' 'bo_kde' 'bo_kde_wrap' 'bo_rf' 'bo_rf_wrap' 'hb' 'hb_wrap' 'bohb' 'bohb_wrap') for (( m=0; m<${#methods[@]}; m++ )) do - nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/usability/pubmed/${methods[$m]}.yaml device $((m%4)) seed ${seed} >/dev/null 2>&1 + nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/usability/pubmed/${methods[$m]}.yaml device $((m%$((m%${device})))) seed ${seed} >/dev/null 2>&1 done From 4b0b52dadec67f988d0d507bc29cf4ab3dadf0e6 Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Mon, 16 Jan 2023 14:41:44 +0800 Subject: [PATCH 28/38] roll back --- scripts/example_configs/femnist/avg/bo_gp_wrap.yaml | 4 ++-- scripts/example_configs/femnist/avg/bo_rf_wrap.yaml | 6 +++--- scripts/example_configs/femnist/avg/bohb_wrap.yaml | 6 +++--- scripts/example_configs/femnist/avg/hb_wrap.yaml | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/scripts/example_configs/femnist/avg/bo_gp_wrap.yaml b/scripts/example_configs/femnist/avg/bo_gp_wrap.yaml index f9bca6368..b7c9896e7 100644 --- a/scripts/example_configs/femnist/avg/bo_gp_wrap.yaml +++ b/scripts/example_configs/femnist/avg/bo_gp_wrap.yaml @@ -7,7 +7,7 @@ seed: 12345 federate: mode: standalone total_round_num: 500 - sample_client_rate: 0.01 + sample_client_rate: 1.0 share_local_model: True online_aggr: True use_diff: True @@ -45,7 +45,7 @@ hpo: num_workers: 0 ss: 'scripts/example_configs/femnist/avg/ss.yaml' sha: - budgets: [ 3, 3 ] + budgets: [ 50, 50 ] elim_rate: 3 iter: 50 fedex: diff --git a/scripts/example_configs/femnist/avg/bo_rf_wrap.yaml b/scripts/example_configs/femnist/avg/bo_rf_wrap.yaml index 2756f178b..7a46cb26c 100644 --- a/scripts/example_configs/femnist/avg/bo_rf_wrap.yaml +++ b/scripts/example_configs/femnist/avg/bo_rf_wrap.yaml @@ -7,7 +7,7 @@ seed: 12345 federate: mode: standalone total_round_num: 500 - sample_client_rate: 0.01 + sample_client_rate: 1.0 share_local_model: True online_aggr: True use_diff: True @@ -45,9 +45,9 @@ hpo: num_workers: 0 ss: 'scripts/example_configs/femnist/avg/ss.yaml' sha: - budgets: [ 3, 3 ] + budgets: [ 50, 50 ] elim_rate: 3 - iter: 10 + iter: 50 fedex: sched: 'aggressive' use: True diff --git a/scripts/example_configs/femnist/avg/bohb_wrap.yaml b/scripts/example_configs/femnist/avg/bohb_wrap.yaml index 76c1e6d5f..83a74fb8c 100644 --- a/scripts/example_configs/femnist/avg/bohb_wrap.yaml +++ b/scripts/example_configs/femnist/avg/bohb_wrap.yaml @@ -7,7 +7,7 @@ seed: 12345 federate: mode: standalone total_round_num: 500 - sample_client_rate: 0.01 + sample_client_rate: 1.0 share_local_model: True online_aggr: True use_diff: True @@ -46,9 +46,9 @@ hpo: init_cand_num: 100 ss: 'scripts/example_configs/femnist/avg/ss.yaml' sha: - budgets: [ 1, 9 ] + budgets: [ 9, 81 ] elim_rate: 3 - iter: 6 + iter: 12 fedex: sched: 'aggressive' use: True diff --git a/scripts/example_configs/femnist/avg/hb_wrap.yaml b/scripts/example_configs/femnist/avg/hb_wrap.yaml index 64fc90889..1cfdeeca6 100644 --- a/scripts/example_configs/femnist/avg/hb_wrap.yaml +++ b/scripts/example_configs/femnist/avg/hb_wrap.yaml @@ -7,7 +7,7 @@ seed: 12345 federate: mode: standalone total_round_num: 500 - sample_client_rate: 0.01 + sample_client_rate: 1.0 share_local_model: True online_aggr: True use_diff: True @@ -46,7 +46,7 @@ hpo: init_cand_num: 100 ss: 'scripts/example_configs/femnist/avg/ss.yaml' sha: - budgets: [ 1, 9 ] + budgets: [ 9, 81 ] elim_rate: 3 iter: 12 table: From 308b0b5032961588dcbda4fc61833a38489602eb Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Mon, 16 Jan 2023 16:28:53 +0800 Subject: [PATCH 29/38] add analisis --- .../{draw_traj.py => analysis.py} | 119 ++++++++++++++++-- scripts/hpo_exp_scripts/parse_exp_results.py | 3 - .../hpo_exp_scripts/uniqueness/cifar10/run.sh | 18 +-- 3 files changed, 116 insertions(+), 24 deletions(-) rename scripts/hpo_exp_scripts/{draw_traj.py => analysis.py} (50%) delete mode 100644 scripts/hpo_exp_scripts/parse_exp_results.py diff --git a/scripts/hpo_exp_scripts/draw_traj.py b/scripts/hpo_exp_scripts/analysis.py similarity index 50% rename from scripts/hpo_exp_scripts/draw_traj.py rename to scripts/hpo_exp_scripts/analysis.py index 5c201c424..aba21457d 100644 --- a/scripts/hpo_exp_scripts/draw_traj.py +++ b/scripts/hpo_exp_scripts/analysis.py @@ -1,3 +1,12 @@ +import os +import yaml +import numpy as np +import matplotlib.pyplot as plt +from tqdm import tqdm + +FONTSIZE = 40 +MARKSIZE = 25 + METHOD = { 'rs': 'RS', 'bo_gp': 'BO_GP', @@ -13,6 +22,12 @@ 'bohb_wrap': 'BOHB+FedEx', } +TRIAL2SEED = { + 't0': 12345, + 't1': 12346, + 't2': 12347, +} + COLORS = [ u'#1f77b4', u'#ff7f0e', u'#2ca02c', u'#d62728', u'#8c564b', u'#9467bd' ] @@ -22,14 +37,6 @@ def parse_logs(root, file_list, eval_key='client_summarized_weighted_avg', suf=''): - import os - import numpy as np - import matplotlib.pyplot as plt - from tqdm import tqdm - - FONTSIZE = 40 - MARKSIZE = 25 - def process(file): history = [] with open(file, 'r') as F: @@ -122,7 +129,95 @@ def process(file): plt.close() -parse_logs(['t0', 't1', 't2'], - list(METHOD.keys()), - eval_key='server_global_eval', - suf='_pubmed_avg') +def args_run_from_scratch(root, + file_list, + eval_key='client_summarized_weighted_avg', + fs_yaml='********', + suf='', + device=8): + def get_args(root, logs): + log_file = os.path.join(root, logs) + arg_key = [] + arg_val = [] + with open(log_file, 'r') as f: + flag = False + cnt = 0 + for line in f.readlines(): + if 'wrap' in log_file: + if 'Winner config_id:' in line: + idx = line.split('Winner config_id: ')[-1] + if idx.endswith(' ') or idx.endswith('\n'): + idx = idx[:-1] + break + else: + if '== HPO Final ==' in line: + flag = True + + if flag: + if line.startswith(' '): + args = [x for x in line.split(' ') if len(x)] + arg_key += args + if line.startswith('0 '): + args = [x for x in line.split(' ') if len(x)][1:] + arg_val += args + + if 'wrap' in log_file: + fedex_yaml = os.path.join(root, f'idx_{idx}_fedex.yaml') + arm_yaml = os.path.join(root, f'{idx}_tmp_grid_search_space.yaml') + + with open(fedex_yaml, 'r') as y: + fedex = dict(yaml.load(y, Loader=yaml.FullLoader)) + + with open(arm_yaml, 'r') as y: + arm = dict(yaml.load(y, Loader=yaml.FullLoader)) + + best_arm = np.argmax(fedex['z'][0]) + arg = '' + for key, val in arm[f'arm{best_arm}'].items(): + arg += f'{key} {val} ' + else: + arg_key = [x for x in arg_key if not x.endswith('\n')] + arg_val = [x for x in arg_val if not x.endswith('\n')] + arg = '' + for key, val in zip(arg_key[:-1], arg_val[:-1]): + arg += f'{key} {val} ' + return arg + + d = 0 + for prefix in root: + for file in file_list: + if 'wrap' in file: + new_name = f'wrap_{file[:-5]}' + else: + new_name = file + + if 'hb' in file: + budget = [9, 81] + else: + budget = [50, 50] + args = get_args( + os.path.join(prefix, f'{file}{suf}'), + os.path.join( + f'{new_name}_{budget}_server_global_eval.val_avg_loss', + 'exp_print.log')) + + seed = TRIAL2SEED[prefix] + print( + f'nohup python federatedscope/main.py --cfg {fs_yaml} {args} device {d % device} > {file}_{seed}{suf}.log &' + ) + d += 1 + + +if __name__ == '__main__': + parse_logs(list(TRIAL2SEED.keys()), + list(METHOD.keys()), + eval_key='server_global_eval', + suf='_pubmed_avg') + + args_run_from_scratch( + list(TRIAL2SEED.keys()), + list(METHOD.keys()), + eval_key='server_global_eval', + fs_yaml= + 'scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml', + suf='_pubmed_avg') diff --git a/scripts/hpo_exp_scripts/parse_exp_results.py b/scripts/hpo_exp_scripts/parse_exp_results.py deleted file mode 100644 index 5d252c689..000000000 --- a/scripts/hpo_exp_scripts/parse_exp_results.py +++ /dev/null @@ -1,3 +0,0 @@ -# TODO: implement this -def parse(): - ... diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh b/scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh index 5bfcca215..7f2928fb1 100644 --- a/scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh @@ -1,17 +1,17 @@ set -e # How to use: -#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 0.05 12345 -#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 0.05 12346 -#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 0.05 12347 +#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 0.05 12345 8 +#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 0.05 12346 8 +#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 0.05 12347 8 # -#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 0.5 12345 -#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 0.5 12346 -#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 0.5 12347 +#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 0.5 12345 8 +#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 0.5 12346 8 +#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 0.5 12347 8 # -#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 5.0 12345 -#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 5.0 12346 -#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 5.0 12347 +#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 5.0 12345 8 +#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 5.0 12346 8 +#bash scripts/hpo_exp_scripts/uniqueness/cifar10/run.sh 5.0 12347 8 alpha=$1 seed=$2 From 6e827e1bc4c529749eff77a68095d8f2c57992a8 Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Mon, 16 Jan 2023 16:30:32 +0800 Subject: [PATCH 30/38] add analisis --- scripts/hpo_exp_scripts/analysis.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/hpo_exp_scripts/analysis.py b/scripts/hpo_exp_scripts/analysis.py index aba21457d..3708c699c 100644 --- a/scripts/hpo_exp_scripts/analysis.py +++ b/scripts/hpo_exp_scripts/analysis.py @@ -131,7 +131,6 @@ def process(file): def args_run_from_scratch(root, file_list, - eval_key='client_summarized_weighted_avg', fs_yaml='********', suf='', device=8): @@ -217,7 +216,6 @@ def get_args(root, logs): args_run_from_scratch( list(TRIAL2SEED.keys()), list(METHOD.keys()), - eval_key='server_global_eval', fs_yaml= 'scripts/hpo_exp_scripts/usability/pubmed/learn_from_scratch/pubmed.yaml', suf='_pubmed_avg') From d0bf0372e5c138dcb715e4add6e6f928f10f3a39 Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Mon, 16 Jan 2023 22:58:48 +0800 Subject: [PATCH 31/38] add learn_from_scratch --- .../attack/learn_from_scratch/attack.yaml | 41 +++++++++++++++++++ .../uniqueness/personalized/run.sh | 2 +- 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 scripts/hpo_exp_scripts/robustness/attack/learn_from_scratch/attack.yaml diff --git a/scripts/hpo_exp_scripts/robustness/attack/learn_from_scratch/attack.yaml b/scripts/hpo_exp_scripts/robustness/attack/learn_from_scratch/attack.yaml new file mode 100644 index 000000000..acbc4dff6 --- /dev/null +++ b/scripts/hpo_exp_scripts/robustness/attack/learn_from_scratch/attack.yaml @@ -0,0 +1,41 @@ +use_gpu: True +device: 0 +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/run.sh b/scripts/hpo_exp_scripts/uniqueness/personalized/run.sh index 36e75e8da..7815fc92e 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/run.sh +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/run.sh @@ -8,5 +8,5 @@ methods=('rs' 'rs_wrap' 'bo_gp' 'bo_gp_wrap' 'bo_kde' 'bo_kde_wrap' 'bo_rf' 'bo_ for (( m=0; m<${#methods[@]}; m++ )) do - nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/uniqueness/personalized/${methods[$m]}.yaml device $((m%${device})) seed ${seed} >/dev/null 2>&1 + nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/uniqueness/personalized/${methods[$m]}.yaml device $((m%${device})) seed ${seed} >/dev/null 2>&1 & done \ No newline at end of file From 6cc10d8221cf07c7cd643b2d376701b84e4503bf Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Mon, 16 Jan 2023 23:50:15 +0800 Subject: [PATCH 32/38] add scipts --- .../learn_from_scratch/cifar10_0.05.yaml | 46 +++++++++++++++++++ .../learn_from_scratch/cifar10_0.5.yaml | 46 +++++++++++++++++++ .../{cifar10.yaml => cifar10_5.0.yaml} | 0 3 files changed, 92 insertions(+) create mode 100644 scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_0.05.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_0.5.yaml rename scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/{cifar10.yaml => cifar10_5.0.yaml} (100%) diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_0.05.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_0.05.yaml new file mode 100644 index 000000000..1a16b7489 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_0.05.yaml @@ -0,0 +1,46 @@ +use_gpu: True +device: 0 +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 0.05}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_0.5.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_0.5.yaml new file mode 100644 index 000000000..f83b476f8 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_0.5.yaml @@ -0,0 +1,46 @@ +use_gpu: True +device: 0 +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True + use_diff: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 0.5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_5.0.yaml similarity index 100% rename from scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10.yaml rename to scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_5.0.yaml From 58d3af4ffeaa5cf5d318e1088adc049425cee38d Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Tue, 17 Jan 2023 12:05:02 +0800 Subject: [PATCH 33/38] update scripts --- scripts/hpo_exp_scripts/usability/pubmed/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/hpo_exp_scripts/usability/pubmed/run.sh b/scripts/hpo_exp_scripts/usability/pubmed/run.sh index b4e3da1bd..ca7aab7ba 100644 --- a/scripts/hpo_exp_scripts/usability/pubmed/run.sh +++ b/scripts/hpo_exp_scripts/usability/pubmed/run.sh @@ -8,5 +8,5 @@ methods=('rs' 'rs_wrap' 'bo_gp' 'bo_gp_wrap' 'bo_kde' 'bo_kde_wrap' 'bo_rf' 'bo_ for (( m=0; m<${#methods[@]}; m++ )) do - nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/usability/pubmed/${methods[$m]}.yaml device $((m%$((m%${device})))) seed ${seed} >/dev/null 2>&1 + nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/usability/pubmed/${methods[$m]}.yaml device $((m%${device})) seed ${seed} >/dev/null 2>&1 done From 81482d0e8a193496357aa7b5434e92888730e8dd Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Tue, 17 Jan 2023 12:19:41 +0800 Subject: [PATCH 34/38] add run --- scripts/hpo_exp_scripts/robustness/attack/run.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/hpo_exp_scripts/robustness/attack/run.sh b/scripts/hpo_exp_scripts/robustness/attack/run.sh index 128d0371e..7fd0554b0 100644 --- a/scripts/hpo_exp_scripts/robustness/attack/run.sh +++ b/scripts/hpo_exp_scripts/robustness/attack/run.sh @@ -1,5 +1,17 @@ set -e +#bash scripts/hpo_exp_scripts/robustness/attack/run.sh "[1,2,3,4]" 1.0 12345 8 +#bash scripts/hpo_exp_scripts/robustness/attack/run.sh "[1,2,3,4]" 1.0 12346 8 +#bash scripts/hpo_exp_scripts/robustness/attack/run.sh "[1,2,3,4]" 1.0 12347 8 +# +#bash scripts/hpo_exp_scripts/robustness/attack/run.sh "[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]" 1.0 12345 8 +#bash scripts/hpo_exp_scripts/robustness/attack/run.sh "[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]" 1.0 12346 8 +#bash scripts/hpo_exp_scripts/robustness/attack/run.sh "[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]" 1.0 12347 8 +# +#bash scripts/hpo_exp_scripts/robustness/attack/run.sh "[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64]" 1.0 12345 8 +#bash scripts/hpo_exp_scripts/robustness/attack/run.sh "[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64]" 1.0 12346 8 +#bash scripts/hpo_exp_scripts/robustness/attack/run.sh "[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64]" 1.0 12347 8 + attacker=$1 sigma=$2 seed=$3 From 1499e0bb78b639ee9558757e3f333c38975d2949 Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Wed, 18 Jan 2023 15:31:15 +0800 Subject: [PATCH 35/38] add run_from_scatch --- client_cfg.yaml | 0 .../personalized/learn_from_scratch/psn.yaml | 40 +++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 client_cfg.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/personalized/learn_from_scratch/psn.yaml diff --git a/client_cfg.yaml b/client_cfg.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/learn_from_scratch/psn.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/learn_from_scratch/psn.yaml new file mode 100644 index 000000000..7ee2ec642 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/learn_from_scratch/psn.yaml @@ -0,0 +1,40 @@ +use_gpu: True +device: 3 +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: femnist + splits: [0.6,0.2,0.2] + subsample: 0.05 + transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] +dataloader: + batch_size: 16 +model: + type: convnet2 + hidden: 2048 + out_channels: 62 + dropout: 0.5 +train: + batch_or_epoch: epoch + local_update_steps: 1 + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] \ No newline at end of file From f2a88b9222de2171ddc881d742b5018497b033f7 Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Wed, 18 Jan 2023 15:32:43 +0800 Subject: [PATCH 36/38] remove empty file --- client_cfg.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 client_cfg.yaml diff --git a/client_cfg.yaml b/client_cfg.yaml deleted file mode 100644 index e69de29bb..000000000 From 1a7862da4739d6d997d989c471def3fe6b6cdf90 Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Fri, 20 Jan 2023 01:21:54 +0800 Subject: [PATCH 37/38] add multi-obj --- federatedscope/autotune/algos.py | 3 +- federatedscope/autotune/run.py | 4 +- federatedscope/autotune/smac.py | 40 ++++++++++++- federatedscope/core/configs/cfg_hpo.py | 7 +++ .../fairness/learn_from_scratch/cifar.yaml.py | 0 .../uniqueness/fairness/multi_obj.yaml | 60 +++++++++++++++++++ .../uniqueness/fairness/run.sh | 34 +++++++++++ .../uniqueness/fairness/test_multi_obj.yaml | 59 ++++++++++++++++++ setup.py | 2 +- 9 files changed, 204 insertions(+), 5 deletions(-) create mode 100644 scripts/hpo_exp_scripts/uniqueness/fairness/learn_from_scratch/cifar.yaml.py create mode 100644 scripts/hpo_exp_scripts/uniqueness/fairness/multi_obj.yaml create mode 100644 scripts/hpo_exp_scripts/uniqueness/fairness/run.sh create mode 100644 scripts/hpo_exp_scripts/uniqueness/fairness/test_multi_obj.yaml diff --git a/federatedscope/autotune/algos.py b/federatedscope/autotune/algos.py index 9d397e9c3..f29b4e607 100644 --- a/federatedscope/autotune/algos.py +++ b/federatedscope/autotune/algos.py @@ -75,7 +75,8 @@ def get_scheduler(init_cfg, client_cfgs=None): # TODO: fix wrap_sha support_optimizer = [ 'sha', 'rs', 'bo_kde', 'hb', 'bohb', 'wrap_rs', 'wrap_bo_kde', - 'wrap_hb', 'wrap_bohb', 'bo_gp', 'bo_rf', 'wrap_bo_gp', 'wrap_bo_rf' + 'wrap_hb', 'wrap_bohb', 'bo_gp', 'bo_rf', 'wrap_bo_gp', 'wrap_bo_rf', + 'multi' ] assert init_cfg.hpo.scheduler in support_optimizer, \ f'`init_cfg.hpo.scheduler` must be one of {support_optimizer}.' diff --git a/federatedscope/autotune/run.py b/federatedscope/autotune/run.py index 0a453f416..c80911d32 100644 --- a/federatedscope/autotune/run.py +++ b/federatedscope/autotune/run.py @@ -17,7 +17,9 @@ def run_scheduler(scheduler, cfg, client_cfgs=None): ]: from federatedscope.autotune.hpbandster import run_hpbandster run_hpbandster(cfg, scheduler, client_cfgs) - elif cfg.hpo.scheduler in ['bo_gp', 'bo_rf', 'wrap_bo_gp', 'wrap_bo_rf']: + elif cfg.hpo.scheduler in [ + 'bo_gp', 'bo_rf', 'wrap_bo_gp', 'wrap_bo_rf', 'multi' + ]: from federatedscope.autotune.smac import run_smac run_smac(cfg, scheduler, client_cfgs) else: diff --git a/federatedscope/autotune/smac.py b/federatedscope/autotune/smac.py index 0092728e8..4facbb86f 100644 --- a/federatedscope/autotune/smac.py +++ b/federatedscope/autotune/smac.py @@ -4,6 +4,7 @@ from smac.facade.smac_bb_facade import SMAC4BB from smac.facade.smac_hpo_facade import SMAC4HPO from smac.scenario.scenario import Scenario +from smac.multi_objective.parego import ParEGO logging.basicConfig(level=logging.WARNING) logger = logging.getLogger(__name__) @@ -32,10 +33,21 @@ def optimization_function_wrapper(config): client_cfgs) key1, key2 = cfg.hpo.metric.split('.') res = results[key1][key2] + if cfg.hpo.scheduler == 'multi': + new_res = {cfg.hpo.metric: res} + for key, w in zip(cfg.hpo.multi_obj.key, cfg.hpo.multi_obj.weight): + key1, key2 = key.split('.') + new_res[key] = w * results[key1][key2] + res = new_res config = dict(config) config['federate.total_round_num'] = budget init_configs.append(config) - perfs.append(res) + + if isinstance(res, dict): + perfs.append(sum(res.values())) + else: + perfs.append(res) + config_id += 1 logger.info(f'Evaluate the {len(perfs)-1}-th config ' f'{config}, and get performance {res}') @@ -65,13 +77,19 @@ def summarize(): np.log(cfg.hpo.sha.budgets[0] / cfg.hpo.sha.budgets[-1]) / np.log(cfg.hpo.sha.elim_rate)) + 1 + if cfg.hpo.scheduler == 'multi': + multi_obj_kwargs = { + "multi_objectives": [cfg.hpo.metric] + cfg.hpo.multi_obj.key, + } + scenario = Scenario({ "run_obj": "quality", "runcount-limit": n_iterations, "cs": config_space, "output_dir": cfg.hpo.working_folder, "deterministic": "true", - "limit_resources": False + "limit_resources": False, + **multi_obj_kwargs }) if cfg.hpo.scheduler.endswith('bo_gp'): @@ -81,6 +99,24 @@ def summarize(): elif cfg.hpo.scheduler.endswith('bo_rf'): smac = SMAC4HPO(scenario=scenario, tae_runner=optimization_function_wrapper) + elif cfg.hpo.scheduler == 'multi': + if cfg.hpo.multi_obj.algo == 'mean': + smac = SMAC4BB( + scenario=scenario, + tae_runner=optimization_function_wrapper, + ) + elif cfg.hpo.multi_obj.algo == 'parego': + smac = SMAC4HPO( + scenario=scenario, + tae_runner=optimization_function_wrapper, + multi_objective_algorithm=ParEGO, + multi_objective_kwargs={ + "rho": 0.05, + }, + ) + else: + raise ValueError(f'Unsupported `cfg.hpo.multi_obj.algo`' + f' {cfg.hpo.multi_obj.algo}') else: raise NotImplementedError try: diff --git a/federatedscope/core/configs/cfg_hpo.py b/federatedscope/core/configs/cfg_hpo.py index f6f41e55f..efbeb88b1 100644 --- a/federatedscope/core/configs/cfg_hpo.py +++ b/federatedscope/core/configs/cfg_hpo.py @@ -18,6 +18,13 @@ def extend_hpo_cfg(cfg): cfg.hpo.scheduler = 'rs' cfg.hpo.metric = 'client_summarized_weighted_avg.val_loss' + # multi-objective-optimization + cfg.hpo.multi_obj = CN() + cfg.hpo.multi_obj.algo = '' # Choose from 'mean', 'parego' + cfg.hpo.multi_obj.key = [] + cfg.hpo.multi_obj.weight = [] # `cfg.hpo.metric` weight is 1.0 when + # multi-objective-optimization algo is `mean` + # SHA cfg.hpo.sha = CN() cfg.hpo.sha.elim_rate = 3 diff --git a/scripts/hpo_exp_scripts/uniqueness/fairness/learn_from_scratch/cifar.yaml.py b/scripts/hpo_exp_scripts/uniqueness/fairness/learn_from_scratch/cifar.yaml.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/hpo_exp_scripts/uniqueness/fairness/multi_obj.yaml b/scripts/hpo_exp_scripts/uniqueness/fairness/multi_obj.yaml new file mode 100644 index 000000000..3843f650c --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/fairness/multi_obj.yaml @@ -0,0 +1,60 @@ +use_gpu: True +device: 3 +outdir: multi_mean_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: multi + num_workers: 0 + multi_obj: + algo: 'mean' + key: ['client_summarized_fairness.val_avg_loss_std'] + weight: [1.0] + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 50, 50 ] + elim_rate: 3 + iter: 50 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: multi_mean_cifar10_avg diff --git a/scripts/hpo_exp_scripts/uniqueness/fairness/run.sh b/scripts/hpo_exp_scripts/uniqueness/fairness/run.sh new file mode 100644 index 000000000..ac399c42d --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/fairness/run.sh @@ -0,0 +1,34 @@ +set -e + +# How to use: +#bash scripts/hpo_exp_scripts/uniqueness/fairness/run.sh 8 + +device=$1 + + +alphas=(0.05 0.5 5.0) +seeds=(12345 12346 12347) +weights=(0.1 1.0 10.0) +trial=0 + +for (( a=0; a<${#alphas[@]}; a++ )) +do + alpha=${alphas[$a]} + for (( s=0; s<${#seeds[@]}; s++ )) + do + seed=${seeds[$s]} + for (( w=0; w<${#weights[@]}; w++ )) + do + weight=${weights[$w]} + folder="${alpha}_${weight}_${seed}_multi_mean_cifar10_avg" + nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/uniqueness/fairness/multi_obj.yaml outdir $folder hpo.working_folder $folder device $((trial%${device})) hpo.multi_obj.weight "[${weight}]" seed ${seed} data.splitter_args "[{'alpha': ${alpha}}]" >/dev/null 2>&1 & + trial=$((trial+1)) + sleep 0.5 + done + + weight=1.0 + folder="${alpha}_${weight}_${seed}_multi_parego_cifar10_avg" + nohup python federatedscope/hpo.py --cfg scripts/hpo_exp_scripts/uniqueness/fairness/multi_obj.yaml outdir $folder hpo.working_folder $folder device $((trial%${device})) hpo.multi_obj.algo parego hpo.multi_obj.weight "[${weight}]" seed ${seed} data.splitter_args "[{'alpha': ${alpha}}]" >/dev/null 2>&1 & + trial=$((trial+1)) + done +done \ No newline at end of file diff --git a/scripts/hpo_exp_scripts/uniqueness/fairness/test_multi_obj.yaml b/scripts/hpo_exp_scripts/uniqueness/fairness/test_multi_obj.yaml new file mode 100644 index 000000000..f4b9d28b8 --- /dev/null +++ b/scripts/hpo_exp_scripts/uniqueness/fairness/test_multi_obj.yaml @@ -0,0 +1,59 @@ +use_gpu: True +device: 3 +outdir: test_multi_mean_cifar10_avg +early_stop: + patience: 100 +seed: 12345 +federate: + mode: standalone + client_num: 5 + total_round_num: 500 + sample_client_rate: 1.0 + share_local_model: True + online_aggr: True +data: + root: data/ + type: 'CIFAR10@torchvision' + splits: [0.8,0.2,0.0] + num_workers: 0 + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + args: [{'download': True}] + splitter: 'lda' + splitter_args: [{'alpha': 5}] +dataloader: + batch_size: 64 +model: + type: convnet2 + hidden: 2048 + out_channels: 10 + dropout: 0.0 +train: + local_update_steps: 1 + batch_or_epoch: epoch + optimizer: + lr: 0.01 + weight_decay: 0.0 +grad: + grad_clip: 5.0 +criterion: + type: CrossEntropyLoss +trainer: + type: cvtrainer +eval: + freq: 1 + metrics: ['acc', 'correct', 'f1'] + split: ['test', 'val', 'train'] +hpo: + scheduler: multi + num_workers: 0 + multi_obj: + algo: 'mean' + key: ['client_summarized_fairness.val_avg_loss_std'] + weight: [1.0] + ss: 'scripts/hpo_exp_scripts/uniqueness/cifar10/ss.yaml' + sha: + budgets: [ 2, 2 ] + iter: 2 + metric: 'client_summarized_weighted_avg.val_avg_loss' + working_folder: test_multi_mean_cifar10_avg diff --git a/setup.py b/setup.py index 892a94318..a8fffb21e 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ ] benchmark_hpo_requires = [ - 'configspace==0.5.0', 'hpbandster==0.7.4', 'smac==1.3.3', 'optuna==2.10.0' + 'configspace==0.5.0', 'hpbandster==0.7.4', 'smac==1.4.0', 'optuna==2.10.0' ] benchmark_htl_requires = ['learn2learn'] From 86a6e8455dca5e26a9774ed334918fdcb7a1bcb5 Mon Sep 17 00:00:00 2001 From: rayrayraykk <18007356109@163.com> Date: Fri, 10 Feb 2023 12:11:06 +0800 Subject: [PATCH 38/38] fix yaml --- .../FedHPOBench/scripts/cnn/cifar10.yaml | 2 +- .../FedHPOBench/scripts/cnn/femnist.yaml | 2 +- .../FedHPOBench/scripts/cnn/femnist_dp.yaml | 2 +- ...oor_badnet_fedavg_convnet2_on_femnist.yaml | 2 +- .../example_configs/cora/hpo_ss_fedex.yaml | 11 ---- .../cora/hpo_ss_fedex_arm.yaml | 12 ----- .../cora/hpo_ss_fedex_arm_table.yaml | 3 -- .../cora/hpo_ss_fedex_grid.yaml | 4 -- scripts/example_configs/cora/hpo_ss_sha.yaml | 14 ----- scripts/example_configs/cora/run.sh | 8 --- scripts/example_configs/cora/sha.yaml | 45 ---------------- .../example_configs/cora/sha_wrap_fedex.yaml | 48 ----------------- .../cora/sha_wrap_fedex_arm.yaml | 54 ------------------- .../example_configs/femnist/hpo_ss_fedex.yaml | 11 ---- .../femnist/hpo_ss_fedex_arm.yaml | 15 ------ .../femnist/hpo_ss_fedex_arm_table.yaml | 3 -- .../femnist/hpo_ss_fedex_grid.yaml | 5 -- .../example_configs/femnist/hpo_ss_sha.yaml | 17 ------ scripts/example_configs/femnist/run.sh | 8 --- scripts/example_configs/femnist/sha.yaml | 49 ----------------- .../femnist/sha_wrap_fedex.yaml | 52 ------------------ .../robustness/attack/bo_gp_wrap.yaml | 2 +- .../robustness/attack/bo_kde_wrap.yaml | 2 +- .../robustness/attack/bo_rf_wrap.yaml | 2 +- .../robustness/attack/bohb_wrap.yaml | 2 +- .../robustness/attack/hb_wrap.yaml | 2 +- .../attack/learn_from_scratch/attack.yaml | 2 +- .../robustness/attack/rs_wrap.yaml | 2 +- .../uniqueness/cifar10/bo_gp.yaml | 3 +- .../uniqueness/cifar10/bo_gp_wrap.yaml | 3 +- .../uniqueness/cifar10/bo_kde.yaml | 3 +- .../uniqueness/cifar10/bo_kde_wrap.yaml | 3 +- .../uniqueness/cifar10/bo_rf.yaml | 3 +- .../uniqueness/cifar10/bo_rf_wrap.yaml | 3 +- .../uniqueness/cifar10/bohb.yaml | 3 +- .../uniqueness/cifar10/bohb_wrap.yaml | 3 +- .../uniqueness/cifar10/hb.yaml | 3 +- .../uniqueness/cifar10/hb_wrap.yaml | 3 +- .../learn_from_scratch/cifar10_0.05.yaml | 3 +- .../learn_from_scratch/cifar10_0.5.yaml | 3 +- .../learn_from_scratch/cifar10_5.0.yaml | 3 +- .../uniqueness/cifar10/rs.yaml | 3 +- .../uniqueness/cifar10/rs_wrap.yaml | 3 +- .../uniqueness/fairness/multi_obj.yaml | 3 +- .../uniqueness/fairness/test_multi_obj.yaml | 3 +- .../uniqueness/personalized/bo_gp.yaml | 2 +- .../uniqueness/personalized/bo_gp_wrap.yaml | 2 +- .../uniqueness/personalized/bo_kde.yaml | 2 +- .../uniqueness/personalized/bo_kde_wrap.yaml | 2 +- .../uniqueness/personalized/bo_rf.yaml | 2 +- .../uniqueness/personalized/bo_rf_wrap.yaml | 2 +- .../uniqueness/personalized/bohb.yaml | 2 +- .../uniqueness/personalized/bohb_wrap.yaml | 2 +- .../uniqueness/personalized/hb.yaml | 2 +- .../uniqueness/personalized/hb_wrap.yaml | 2 +- .../personalized/learn_from_scratch/psn.yaml | 2 +- .../uniqueness/personalized/rs.yaml | 2 +- .../uniqueness/personalized/rs_wrap.yaml | 2 +- .../usability/femnist}/bo_gp.yaml | 2 +- .../usability/femnist}/bo_gp_wrap.yaml | 2 +- .../usability/femnist}/bo_kde.yaml | 2 +- .../usability/femnist}/bo_kde_wrap.yaml | 2 +- .../usability/femnist}/bo_rf.yaml | 2 +- .../usability/femnist}/bo_rf_wrap.yaml | 2 +- .../usability/femnist}/bohb.yaml | 2 +- .../usability/femnist}/bohb_wrap.yaml | 2 +- .../usability/femnist}/hb.yaml | 2 +- .../usability/femnist}/hb_wrap.yaml | 2 +- .../femnist}/learn_from_scratch.yaml | 2 +- .../usability/femnist}/rs.yaml | 2 +- .../usability/femnist}/rs_wrap.yaml | 2 +- .../usability/femnist}/rs_wrap_pfedex.sh | 0 .../usability/femnist}/sha.yaml | 2 +- .../usability/femnist}/sha_wrap.yaml | 2 +- .../usability/femnist}/ss.yaml | 0 scripts/tianchi/submit.yaml | 2 +- 76 files changed, 57 insertions(+), 433 deletions(-) delete mode 100644 scripts/example_configs/cora/hpo_ss_fedex.yaml delete mode 100644 scripts/example_configs/cora/hpo_ss_fedex_arm.yaml delete mode 100644 scripts/example_configs/cora/hpo_ss_fedex_arm_table.yaml delete mode 100644 scripts/example_configs/cora/hpo_ss_fedex_grid.yaml delete mode 100644 scripts/example_configs/cora/hpo_ss_sha.yaml delete mode 100644 scripts/example_configs/cora/run.sh delete mode 100644 scripts/example_configs/cora/sha.yaml delete mode 100644 scripts/example_configs/cora/sha_wrap_fedex.yaml delete mode 100644 scripts/example_configs/cora/sha_wrap_fedex_arm.yaml delete mode 100644 scripts/example_configs/femnist/hpo_ss_fedex.yaml delete mode 100644 scripts/example_configs/femnist/hpo_ss_fedex_arm.yaml delete mode 100644 scripts/example_configs/femnist/hpo_ss_fedex_arm_table.yaml delete mode 100644 scripts/example_configs/femnist/hpo_ss_fedex_grid.yaml delete mode 100644 scripts/example_configs/femnist/hpo_ss_sha.yaml delete mode 100644 scripts/example_configs/femnist/run.sh delete mode 100644 scripts/example_configs/femnist/sha.yaml delete mode 100644 scripts/example_configs/femnist/sha_wrap_fedex.yaml rename scripts/{example_configs/femnist/avg => hpo_exp_scripts/usability/femnist}/bo_gp.yaml (91%) rename scripts/{example_configs/femnist/avg => hpo_exp_scripts/usability/femnist}/bo_gp_wrap.yaml (92%) rename scripts/{example_configs/femnist/avg => hpo_exp_scripts/usability/femnist}/bo_kde.yaml (91%) rename scripts/{example_configs/femnist/avg => hpo_exp_scripts/usability/femnist}/bo_kde_wrap.yaml (93%) rename scripts/{example_configs/femnist/avg => hpo_exp_scripts/usability/femnist}/bo_rf.yaml (91%) rename scripts/{example_configs/femnist/avg => hpo_exp_scripts/usability/femnist}/bo_rf_wrap.yaml (92%) rename scripts/{example_configs/femnist/avg => hpo_exp_scripts/usability/femnist}/bohb.yaml (91%) rename scripts/{example_configs/femnist/avg => hpo_exp_scripts/usability/femnist}/bohb_wrap.yaml (92%) rename scripts/{example_configs/femnist/avg => hpo_exp_scripts/usability/femnist}/hb.yaml (91%) rename scripts/{example_configs/femnist/avg => hpo_exp_scripts/usability/femnist}/hb_wrap.yaml (93%) rename scripts/{example_configs/femnist/avg => hpo_exp_scripts/usability/femnist}/learn_from_scratch.yaml (89%) rename scripts/{example_configs/femnist/avg => hpo_exp_scripts/usability/femnist}/rs.yaml (92%) rename scripts/{example_configs/femnist/avg => hpo_exp_scripts/usability/femnist}/rs_wrap.yaml (93%) rename scripts/{example_configs/femnist/avg => hpo_exp_scripts/usability/femnist}/rs_wrap_pfedex.sh (100%) rename scripts/{example_configs/femnist/avg => hpo_exp_scripts/usability/femnist}/sha.yaml (91%) rename scripts/{example_configs/femnist/avg => hpo_exp_scripts/usability/femnist}/sha_wrap.yaml (92%) rename scripts/{example_configs/femnist/avg => hpo_exp_scripts/usability/femnist}/ss.yaml (100%) diff --git a/benchmark/FedHPOBench/scripts/cnn/cifar10.yaml b/benchmark/FedHPOBench/scripts/cnn/cifar10.yaml index c2514c62e..224875cdc 100644 --- a/benchmark/FedHPOBench/scripts/cnn/cifar10.yaml +++ b/benchmark/FedHPOBench/scripts/cnn/cifar10.yaml @@ -17,7 +17,7 @@ data: splits: [0.8,0.2,0.0] batch_size: 32 num_workers: 0 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] args: [{'download': True}] splitter: 'lda' splitter_args: [{'alpha': 0.5}] diff --git a/benchmark/FedHPOBench/scripts/cnn/femnist.yaml b/benchmark/FedHPOBench/scripts/cnn/femnist.yaml index c64356b44..58312f120 100644 --- a/benchmark/FedHPOBench/scripts/cnn/femnist.yaml +++ b/benchmark/FedHPOBench/scripts/cnn/femnist.yaml @@ -17,7 +17,7 @@ data: splits: [0.6,0.2,0.2] batch_size: 16 subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] num_workers: 0 model: type: convnet2 diff --git a/benchmark/FedHPOBench/scripts/cnn/femnist_dp.yaml b/benchmark/FedHPOBench/scripts/cnn/femnist_dp.yaml index d422df2bd..8dbe21f2b 100644 --- a/benchmark/FedHPOBench/scripts/cnn/femnist_dp.yaml +++ b/benchmark/FedHPOBench/scripts/cnn/femnist_dp.yaml @@ -28,7 +28,7 @@ data: splits: [0.6,0.2,0.2] batch_size: 16 subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] num_workers: 0 model: type: convnet2 diff --git a/scripts/attack_exp_scripts/backdoor_attack/backdoor_badnet_fedavg_convnet2_on_femnist.yaml b/scripts/attack_exp_scripts/backdoor_attack/backdoor_badnet_fedavg_convnet2_on_femnist.yaml index 09430d4d8..850676fb2 100644 --- a/scripts/attack_exp_scripts/backdoor_attack/backdoor_badnet_fedavg_convnet2_on_femnist.yaml +++ b/scripts/attack_exp_scripts/backdoor_attack/backdoor_badnet_fedavg_convnet2_on_femnist.yaml @@ -17,7 +17,7 @@ data: # form: dataloader splits: [0.6,0.2,0.2] subsample: 0.05 - # transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + # transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] transform: [ [ 'ToTensor' ] ] dataloader: batch_size: 32 diff --git a/scripts/example_configs/cora/hpo_ss_fedex.yaml b/scripts/example_configs/cora/hpo_ss_fedex.yaml deleted file mode 100644 index edece27da..000000000 --- a/scripts/example_configs/cora/hpo_ss_fedex.yaml +++ /dev/null @@ -1,11 +0,0 @@ -hpo.fedex.eta0: - type: cate - choices: [-1.0, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0] -hpo.fedex.gamma: - type: float - lower: 0.0 - upper: 1.0 - log: False -hpo.fedex.diff: - type: cate - choices: [True, False] \ No newline at end of file diff --git a/scripts/example_configs/cora/hpo_ss_fedex_arm.yaml b/scripts/example_configs/cora/hpo_ss_fedex_arm.yaml deleted file mode 100644 index 3b52d0c79..000000000 --- a/scripts/example_configs/cora/hpo_ss_fedex_arm.yaml +++ /dev/null @@ -1,12 +0,0 @@ -train.optimizer.lr: - type: cate - choices: [0.01, 0.01668, 0.02783, 0.04642, 0.07743, 0.12915, 0.21544, 0.35938, 0.59948, 1.0] -train.optimizer.weight_decay: - type: cate - choices: [0.0, 0.001, 0.01, 0.1] -model.dropout: - type: cate - choices: [0.0, 0.5] -train.local_update_steps: - type: cate - choices: [1, 2, 3, 4, 5, 6, 7, 8] \ No newline at end of file diff --git a/scripts/example_configs/cora/hpo_ss_fedex_arm_table.yaml b/scripts/example_configs/cora/hpo_ss_fedex_arm_table.yaml deleted file mode 100644 index d40abc429..000000000 --- a/scripts/example_configs/cora/hpo_ss_fedex_arm_table.yaml +++ /dev/null @@ -1,3 +0,0 @@ -hpo.table.idx: - type: cate - choices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80] diff --git a/scripts/example_configs/cora/hpo_ss_fedex_grid.yaml b/scripts/example_configs/cora/hpo_ss_fedex_grid.yaml deleted file mode 100644 index 4f9699dcf..000000000 --- a/scripts/example_configs/cora/hpo_ss_fedex_grid.yaml +++ /dev/null @@ -1,4 +0,0 @@ -train.optimizer.lr: [0.01, 0.01668, 0.02783, 0.04642, 0.07743, 0.12915, 0.21544, 0.35938, 0.59948, 1.0] -train.optimizer.weight_decay: [0.0, 0.001, 0.01, 0.1] -model.dropout: [0.0, 0.5] -train.local_update_steps: [1, 2, 3, 4, 5, 6, 7, 8] \ No newline at end of file diff --git a/scripts/example_configs/cora/hpo_ss_sha.yaml b/scripts/example_configs/cora/hpo_ss_sha.yaml deleted file mode 100644 index 5928eb21c..000000000 --- a/scripts/example_configs/cora/hpo_ss_sha.yaml +++ /dev/null @@ -1,14 +0,0 @@ -train.optimizer.lr: - type: float - lower: 0.01 - upper: 1.0 - log: True -train.optimizer.weight_decay: - type: cate - choices: [0.0, 0.001, 0.01, 0.1] -model.dropout: - type: cate - choices: [0.0, 0.5] -train.local_update_steps: - type: cate - choices: [1, 2, 3, 4, 5, 6, 7, 8] \ No newline at end of file diff --git a/scripts/example_configs/cora/run.sh b/scripts/example_configs/cora/run.sh deleted file mode 100644 index 3d57fb137..000000000 --- a/scripts/example_configs/cora/run.sh +++ /dev/null @@ -1,8 +0,0 @@ -# SHA -python hpo.py --cfg scripts/example_configs/cora/sha.yaml - -# SHA wrap FedEX (FedEX related param) -python hpo.py --cfg scripts/example_configs/cora/sha_wrap_fedex.yaml - -# SHA wrap FedEX (arm) -python hpo.py --cfg scripts/example_configs/cora/sha_wrap_fedex_arm.yaml \ No newline at end of file diff --git a/scripts/example_configs/cora/sha.yaml b/scripts/example_configs/cora/sha.yaml deleted file mode 100644 index 00444abb6..000000000 --- a/scripts/example_configs/cora/sha.yaml +++ /dev/null @@ -1,45 +0,0 @@ -use_gpu: True -device: 3 -early_stop: - patience: 100 -seed: 12345 -federate: - mode: standalone - make_global_eval: True - client_num: 5 - total_round_num: 500 - share_local_model: True - online_aggr: True - use_diff: True -data: - root: data/ - type: cora - splitter: 'louvain' -dataloader: - batch_size: 1 -model: - type: gcn - hidden: 64 - dropout: 0.5 - out_channels: 7 -train: - local_update_steps: 1 - optimizer: - lr: 0.25 - weight_decay: 0.0005 -criterion: - type: CrossEntropyLoss -trainer: - type: nodefullbatch_trainer -eval: - freq: 1 - metrics: ['acc', 'correct', 'f1'] - split: ['test', 'val', 'train'] -hpo: - scheduler: sha - num_workers: 0 - init_cand_num: 81 - ss: 'scripts/example_configs/cora/hpo_ss_sha.yaml' - sha: - budgets: [2, 4, 12, 36] - metric: 'server_global_eval.val_avg_loss' diff --git a/scripts/example_configs/cora/sha_wrap_fedex.yaml b/scripts/example_configs/cora/sha_wrap_fedex.yaml deleted file mode 100644 index a5c729c14..000000000 --- a/scripts/example_configs/cora/sha_wrap_fedex.yaml +++ /dev/null @@ -1,48 +0,0 @@ -use_gpu: True -device: 3 -early_stop: - patience: 100 -seed: 12345 -federate: - mode: standalone - make_global_eval: True - client_num: 5 - total_round_num: 500 - share_local_model: True - online_aggr: True - use_diff: True -data: - root: data/ - type: cora - splitter: 'louvain' -dataloader: - batch_size: 1 -model: - type: gcn - hidden: 64 - dropout: 0.5 - out_channels: 7 -train: - local_update_steps: 1 - optimizer: - lr: 0.25 - weight_decay: 0.0005 -criterion: - type: CrossEntropyLoss -trainer: - type: nodefullbatch_trainer -eval: - freq: 1 - metrics: ['acc', 'correct', 'f1'] - split: ['test', 'val', 'train'] -hpo: - scheduler: sha - num_workers: 0 - init_cand_num: 81 - ss: 'scripts/example_configs/cora/hpo_ss_fedex.yaml' - sha: - budgets: [2, 4, 12, 36] - fedex: - use: True - ss: 'scripts/example_configs/cora/hpo_ss_fedex_grid.yaml' - metric: 'server_global_eval.val_avg_loss' diff --git a/scripts/example_configs/cora/sha_wrap_fedex_arm.yaml b/scripts/example_configs/cora/sha_wrap_fedex_arm.yaml deleted file mode 100644 index 6566791d5..000000000 --- a/scripts/example_configs/cora/sha_wrap_fedex_arm.yaml +++ /dev/null @@ -1,54 +0,0 @@ -use_gpu: True -device: 3 -early_stop: - patience: 100 -seed: 12345 -federate: - mode: standalone - make_global_eval: True - client_num: 5 - total_round_num: 500 - share_local_model: True - online_aggr: True - use_diff: True -data: - root: data/ - type: cora - splitter: 'louvain' -dataloader: - batch_size: 1 -model: - type: gcn - hidden: 64 - dropout: 0.5 - out_channels: 7 -train: - local_update_steps: 1 - optimizer: - lr: 0.25 - weight_decay: 0.0005 -criterion: - type: CrossEntropyLoss -trainer: - type: nodefullbatch_trainer -eval: - freq: 1 - metrics: ['acc', 'correct', 'f1'] - split: ['test', 'val', 'train'] -hpo: - scheduler: wrap_sha - num_workers: 0 - init_cand_num: 81 - ss: 'scripts/example_configs/cora/hpo_ss_fedex_arm_table.yaml' - table: - ss: 'scripts/example_configs/cora/hpo_ss_fedex_arm.yaml' - num: 4 - cand: 81 - sha: - budgets: [2, 4, 12, 36] - fedex: - use: True - diff: False - eta0: 0.050 - gamma: 0.495861 - metric: 'server_global_eval.val_avg_loss' diff --git a/scripts/example_configs/femnist/hpo_ss_fedex.yaml b/scripts/example_configs/femnist/hpo_ss_fedex.yaml deleted file mode 100644 index edece27da..000000000 --- a/scripts/example_configs/femnist/hpo_ss_fedex.yaml +++ /dev/null @@ -1,11 +0,0 @@ -hpo.fedex.eta0: - type: cate - choices: [-1.0, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0] -hpo.fedex.gamma: - type: float - lower: 0.0 - upper: 1.0 - log: False -hpo.fedex.diff: - type: cate - choices: [True, False] \ No newline at end of file diff --git a/scripts/example_configs/femnist/hpo_ss_fedex_arm.yaml b/scripts/example_configs/femnist/hpo_ss_fedex_arm.yaml deleted file mode 100644 index ea438a1ad..000000000 --- a/scripts/example_configs/femnist/hpo_ss_fedex_arm.yaml +++ /dev/null @@ -1,15 +0,0 @@ -train.optimizer.lr: - type: cate - choices: [0.01, 0.01668, 0.02783, 0.04642, 0.07743, 0.12915, 0.21544, 0.35938, 0.59948, 1.0] -train.optimizer.weight_decay: - type: cate - choices: [0.0, 0.001, 0.01, 0.1] -model.dropout: - type: cate - choices: [0.0, 0.5] -train.local_update_steps: - type: cate - choices: [1, 2, 3, 4] -dataloader.batch_size: - type: cate - choices: [16, 32, 64] \ No newline at end of file diff --git a/scripts/example_configs/femnist/hpo_ss_fedex_arm_table.yaml b/scripts/example_configs/femnist/hpo_ss_fedex_arm_table.yaml deleted file mode 100644 index d40abc429..000000000 --- a/scripts/example_configs/femnist/hpo_ss_fedex_arm_table.yaml +++ /dev/null @@ -1,3 +0,0 @@ -hpo.table.idx: - type: cate - choices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80] diff --git a/scripts/example_configs/femnist/hpo_ss_fedex_grid.yaml b/scripts/example_configs/femnist/hpo_ss_fedex_grid.yaml deleted file mode 100644 index 774265c55..000000000 --- a/scripts/example_configs/femnist/hpo_ss_fedex_grid.yaml +++ /dev/null @@ -1,5 +0,0 @@ -train.optimizer.lr: [0.01, 0.01668, 0.02783, 0.04642, 0.07743, 0.12915, 0.21544, 0.35938, 0.59948, 1.0] -train.optimizer.weight_decay: [0.0, 0.001, 0.01, 0.1] -model.dropout: [0.0, 0.5] -train.local_update_steps: [1, 2, 3, 4] -dataloader.batch_size: [16, 32, 64] \ No newline at end of file diff --git a/scripts/example_configs/femnist/hpo_ss_sha.yaml b/scripts/example_configs/femnist/hpo_ss_sha.yaml deleted file mode 100644 index 4a65f2cef..000000000 --- a/scripts/example_configs/femnist/hpo_ss_sha.yaml +++ /dev/null @@ -1,17 +0,0 @@ -train.optimizer.lr: - type: float - lower: 0.01 - upper: 1.0 - log: True -train.optimizer.weight_decay: - type: cate - choices: [0.0, 0.001, 0.01, 0.1] -model.dropout: - type: cate - choices: [0.0, 0.5] -train.local_update_steps: - type: cate - choices: [1, 2, 3, 4] -dataloader.batch_size: - type: cate - choices: [16, 32, 64] \ No newline at end of file diff --git a/scripts/example_configs/femnist/run.sh b/scripts/example_configs/femnist/run.sh deleted file mode 100644 index 803e70cc9..000000000 --- a/scripts/example_configs/femnist/run.sh +++ /dev/null @@ -1,8 +0,0 @@ -# SHA -python hpo.py --cfg scripts/example_configs/femnist/sha.yaml - -# SHA wrap FedEX (FedEX related param) -python hpo.py --cfg scripts/example_configs/femnist/sha_wrap_fedex.yaml - -# SHA wrap FedEX (arm) -python hpo.py --cfg scripts/example_configs/femnist/sha_wrap_fedex_arm.yaml \ No newline at end of file diff --git a/scripts/example_configs/femnist/sha.yaml b/scripts/example_configs/femnist/sha.yaml deleted file mode 100644 index 2343d43d6..000000000 --- a/scripts/example_configs/femnist/sha.yaml +++ /dev/null @@ -1,49 +0,0 @@ -use_gpu: True -device: 3 -early_stop: - patience: 100 -seed: 12345 -federate: - mode: standalone - total_round_num: 500 - sample_client_rate: 1.0 - share_local_model: True - online_aggr: True - use_diff: True -data: - root: data/ - type: femnist - splits: [0.6,0.2,0.2] - subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] -dataloader: - batch_size: 16 -model: - type: convnet2 - hidden: 2048 - out_channels: 62 - dropout: 0.5 -train: - batch_or_epoch: epoch - local_update_steps: 1 - optimizer: - lr: 0.01 - weight_decay: 0.0 -grad: - grad_clip: 5.0 -criterion: - type: CrossEntropyLoss -trainer: - type: cvtrainer -eval: - freq: 1 - metrics: ['acc', 'correct', 'f1'] - split: ['test', 'val', 'train'] -hpo: - scheduler: sha - num_workers: 0 - init_cand_num: 27 - ss: 'scripts/example_configs/femnist/hpo_ss_sha.yaml' - sha: - budgets: [12, 13, 19] - metric: 'client_summarized_weighted_avg.val_avg_loss' diff --git a/scripts/example_configs/femnist/sha_wrap_fedex.yaml b/scripts/example_configs/femnist/sha_wrap_fedex.yaml deleted file mode 100644 index 58309fe31..000000000 --- a/scripts/example_configs/femnist/sha_wrap_fedex.yaml +++ /dev/null @@ -1,52 +0,0 @@ -use_gpu: True -device: 3 -early_stop: - patience: 100 -seed: 12345 -federate: - mode: standalone - total_round_num: 500 - sample_client_rate: 1.0 - share_local_model: True - online_aggr: True - use_diff: True -data: - root: data/ - type: femnist - splits: [0.6,0.2,0.2] - subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] -dataloader: - batch_size: 16 -model: - type: convnet2 - hidden: 2048 - out_channels: 62 - dropout: 0.5 -train: - batch_or_epoch: epoch - local_update_steps: 1 - optimizer: - lr: 0.01 - weight_decay: 0.0 -grad: - grad_clip: 5.0 -criterion: - type: CrossEntropyLoss -trainer: - type: cvtrainer -eval: - freq: 1 - metrics: ['acc', 'correct', 'f1'] - split: ['test', 'val', 'train'] -hpo: - scheduler: sha - num_workers: 0 - init_cand_num: 81 - ss: 'scripts/example_configs/femnist/hpo_ss_fedex.yaml' - sha: - budgets: [2, 4, 12, 36] - fedex: - use: True - ss: 'scripts/example_configs/femnist/hpo_ss_fedex_grid.yaml' - metric: 'client_summarized_weighted_avg.val_avg_loss' diff --git a/scripts/hpo_exp_scripts/robustness/attack/bo_gp_wrap.yaml b/scripts/hpo_exp_scripts/robustness/attack/bo_gp_wrap.yaml index e5aaa4afd..f7e7beea8 100644 --- a/scripts/hpo_exp_scripts/robustness/attack/bo_gp_wrap.yaml +++ b/scripts/hpo_exp_scripts/robustness/attack/bo_gp_wrap.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/hpo_exp_scripts/robustness/attack/bo_kde_wrap.yaml b/scripts/hpo_exp_scripts/robustness/attack/bo_kde_wrap.yaml index a365f7956..38bce0697 100644 --- a/scripts/hpo_exp_scripts/robustness/attack/bo_kde_wrap.yaml +++ b/scripts/hpo_exp_scripts/robustness/attack/bo_kde_wrap.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/hpo_exp_scripts/robustness/attack/bo_rf_wrap.yaml b/scripts/hpo_exp_scripts/robustness/attack/bo_rf_wrap.yaml index 81f29aa2e..675fd0e1a 100644 --- a/scripts/hpo_exp_scripts/robustness/attack/bo_rf_wrap.yaml +++ b/scripts/hpo_exp_scripts/robustness/attack/bo_rf_wrap.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/hpo_exp_scripts/robustness/attack/bohb_wrap.yaml b/scripts/hpo_exp_scripts/robustness/attack/bohb_wrap.yaml index 5bfb9c834..2caffeb2c 100644 --- a/scripts/hpo_exp_scripts/robustness/attack/bohb_wrap.yaml +++ b/scripts/hpo_exp_scripts/robustness/attack/bohb_wrap.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/hpo_exp_scripts/robustness/attack/hb_wrap.yaml b/scripts/hpo_exp_scripts/robustness/attack/hb_wrap.yaml index 56794c4f0..07504656f 100644 --- a/scripts/hpo_exp_scripts/robustness/attack/hb_wrap.yaml +++ b/scripts/hpo_exp_scripts/robustness/attack/hb_wrap.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/hpo_exp_scripts/robustness/attack/learn_from_scratch/attack.yaml b/scripts/hpo_exp_scripts/robustness/attack/learn_from_scratch/attack.yaml index acbc4dff6..1435cd8dc 100644 --- a/scripts/hpo_exp_scripts/robustness/attack/learn_from_scratch/attack.yaml +++ b/scripts/hpo_exp_scripts/robustness/attack/learn_from_scratch/attack.yaml @@ -15,7 +15,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/hpo_exp_scripts/robustness/attack/rs_wrap.yaml b/scripts/hpo_exp_scripts/robustness/attack/rs_wrap.yaml index 06ea2b67e..e9c852614 100644 --- a/scripts/hpo_exp_scripts/robustness/attack/rs_wrap.yaml +++ b/scripts/hpo_exp_scripts/robustness/attack/rs_wrap.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_gp.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_gp.yaml index c703a1abb..50a2057b9 100644 --- a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_gp.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_gp.yaml @@ -16,8 +16,7 @@ data: type: 'CIFAR10@torchvision' splits: [0.8,0.2,0.0] num_workers: 0 - transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] - test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] args: [{'download': True}] splitter: 'lda' splitter_args: [{'alpha': 5}] diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_gp_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_gp_wrap.yaml index 867d0424f..f454522ce 100644 --- a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_gp_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_gp_wrap.yaml @@ -17,8 +17,7 @@ data: type: 'CIFAR10@torchvision' splits: [0.8,0.2,0.0] num_workers: 0 - transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] - test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] args: [{'download': True}] splitter: 'lda' splitter_args: [{'alpha': 5}] diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_kde.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_kde.yaml index c21d7da72..935e22a9e 100644 --- a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_kde.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_kde.yaml @@ -16,8 +16,7 @@ data: type: 'CIFAR10@torchvision' splits: [0.8,0.2,0.0] num_workers: 0 - transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] - test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] args: [{'download': True}] splitter: 'lda' splitter_args: [{'alpha': 5}] diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_kde_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_kde_wrap.yaml index 5c57fa254..bc1afc4fb 100644 --- a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_kde_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_kde_wrap.yaml @@ -17,8 +17,7 @@ data: type: 'CIFAR10@torchvision' splits: [0.8,0.2,0.0] num_workers: 0 - transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] - test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] args: [{'download': True}] splitter: 'lda' splitter_args: [{'alpha': 5}] diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_rf.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_rf.yaml index cf11c2178..faac26886 100644 --- a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_rf.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_rf.yaml @@ -16,8 +16,7 @@ data: type: 'CIFAR10@torchvision' splits: [0.8,0.2,0.0] num_workers: 0 - transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] - test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] args: [{'download': True}] splitter: 'lda' splitter_args: [{'alpha': 5}] diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_rf_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_rf_wrap.yaml index 83808b669..759082840 100644 --- a/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_rf_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bo_rf_wrap.yaml @@ -17,8 +17,7 @@ data: type: 'CIFAR10@torchvision' splits: [0.8,0.2,0.0] num_workers: 0 - transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] - test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] args: [{'download': True}] splitter: 'lda' splitter_args: [{'alpha': 5}] diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bohb.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bohb.yaml index cd820c13e..bd5107ce5 100644 --- a/scripts/hpo_exp_scripts/uniqueness/cifar10/bohb.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bohb.yaml @@ -16,8 +16,7 @@ data: type: 'CIFAR10@torchvision' splits: [0.8,0.2,0.0] num_workers: 0 - transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] - test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] args: [{'download': True}] splitter: 'lda' splitter_args: [{'alpha': 5}] diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/bohb_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/bohb_wrap.yaml index df3c80eb4..580c99f1d 100644 --- a/scripts/hpo_exp_scripts/uniqueness/cifar10/bohb_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/bohb_wrap.yaml @@ -17,8 +17,7 @@ data: type: 'CIFAR10@torchvision' splits: [0.8,0.2,0.0] num_workers: 0 - transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] - test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] args: [{'download': True}] splitter: 'lda' splitter_args: [{'alpha': 5}] diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/hb.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/hb.yaml index 180e41bf5..6ad5acf72 100644 --- a/scripts/hpo_exp_scripts/uniqueness/cifar10/hb.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/hb.yaml @@ -16,8 +16,7 @@ data: type: 'CIFAR10@torchvision' splits: [0.8,0.2,0.0] num_workers: 0 - transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] - test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] args: [{'download': True}] splitter: 'lda' splitter_args: [{'alpha': 5}] diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/hb_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/hb_wrap.yaml index a1dfe0835..709776bbe 100644 --- a/scripts/hpo_exp_scripts/uniqueness/cifar10/hb_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/hb_wrap.yaml @@ -17,8 +17,7 @@ data: type: 'CIFAR10@torchvision' splits: [0.8,0.2,0.0] num_workers: 0 - transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] - test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] args: [{'download': True}] splitter: 'lda' splitter_args: [{'alpha': 5}] diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_0.05.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_0.05.yaml index 1a16b7489..c9882a1bb 100644 --- a/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_0.05.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_0.05.yaml @@ -16,8 +16,7 @@ data: type: 'CIFAR10@torchvision' splits: [0.8,0.2,0.0] num_workers: 0 - transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] - test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] args: [{'download': True}] splitter: 'lda' splitter_args: [{'alpha': 0.05}] diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_0.5.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_0.5.yaml index f83b476f8..1f70a6c05 100644 --- a/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_0.5.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_0.5.yaml @@ -16,8 +16,7 @@ data: type: 'CIFAR10@torchvision' splits: [0.8,0.2,0.0] num_workers: 0 - transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] - test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] args: [{'download': True}] splitter: 'lda' splitter_args: [{'alpha': 0.5}] diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_5.0.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_5.0.yaml index 21026a3e5..ebd3225ec 100644 --- a/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_5.0.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/learn_from_scratch/cifar10_5.0.yaml @@ -16,8 +16,7 @@ data: type: 'CIFAR10@torchvision' splits: [0.8,0.2,0.0] num_workers: 0 - transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] - test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] args: [{'download': True}] splitter: 'lda' splitter_args: [{'alpha': 5}] diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/rs.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/rs.yaml index f4e073f4c..d765de21c 100644 --- a/scripts/hpo_exp_scripts/uniqueness/cifar10/rs.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/rs.yaml @@ -16,8 +16,7 @@ data: type: 'CIFAR10@torchvision' splits: [0.8,0.2,0.0] num_workers: 0 - transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] - test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] args: [{'download': True}] splitter: 'lda' splitter_args: [{'alpha': 5}] diff --git a/scripts/hpo_exp_scripts/uniqueness/cifar10/rs_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/cifar10/rs_wrap.yaml index e674f98ed..29342770e 100644 --- a/scripts/hpo_exp_scripts/uniqueness/cifar10/rs_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/cifar10/rs_wrap.yaml @@ -17,8 +17,7 @@ data: type: 'CIFAR10@torchvision' splits: [0.8,0.2,0.0] num_workers: 0 - transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] - test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] args: [{'download': True}] splitter: 'lda' splitter_args: [{'alpha': 5}] diff --git a/scripts/hpo_exp_scripts/uniqueness/fairness/multi_obj.yaml b/scripts/hpo_exp_scripts/uniqueness/fairness/multi_obj.yaml index 3843f650c..445480bbb 100644 --- a/scripts/hpo_exp_scripts/uniqueness/fairness/multi_obj.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/fairness/multi_obj.yaml @@ -16,8 +16,7 @@ data: type: 'CIFAR10@torchvision' splits: [0.8,0.2,0.0] num_workers: 0 - transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] - test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] args: [{'download': True}] splitter: 'lda' splitter_args: [{'alpha': 5}] diff --git a/scripts/hpo_exp_scripts/uniqueness/fairness/test_multi_obj.yaml b/scripts/hpo_exp_scripts/uniqueness/fairness/test_multi_obj.yaml index f4b9d28b8..2e8bc1e3a 100644 --- a/scripts/hpo_exp_scripts/uniqueness/fairness/test_multi_obj.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/fairness/test_multi_obj.yaml @@ -16,8 +16,7 @@ data: type: 'CIFAR10@torchvision' splits: [0.8,0.2,0.0] num_workers: 0 - transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] - test_transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.247, 0.243, 0.261]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.4914, 0.4822, 0.4465], 'std': [0.2470, 0.2435, 0.2616]}]] args: [{'download': True}] splitter: 'lda' splitter_args: [{'alpha': 5}] diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp.yaml index e74b3bb68..b87e8b5d8 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp.yaml @@ -15,7 +15,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp_wrap.yaml index b447f72bb..f4add5608 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_gp_wrap.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde.yaml index ac137084a..2b29f75ae 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde.yaml @@ -15,7 +15,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde_wrap.yaml index d90a5c5f8..2edb6d618 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_kde_wrap.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf.yaml index 783fb28f0..60206cd5d 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf.yaml @@ -15,7 +15,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf_wrap.yaml index a0abd0f25..66703064f 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bo_rf_wrap.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bohb.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bohb.yaml index d18031101..394d9be57 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/bohb.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bohb.yaml @@ -15,7 +15,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/bohb_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/bohb_wrap.yaml index d4d374b7a..31354e834 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/bohb_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/bohb_wrap.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/hb.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/hb.yaml index f50553dca..919f6426f 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/hb.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/hb.yaml @@ -15,7 +15,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/hb_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/hb_wrap.yaml index d7605598a..b6a441699 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/hb_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/hb_wrap.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/learn_from_scratch/psn.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/learn_from_scratch/psn.yaml index 7ee2ec642..03219db30 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/learn_from_scratch/psn.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/learn_from_scratch/psn.yaml @@ -14,7 +14,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/rs.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/rs.yaml index 06a994890..b52b27c5e 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/rs.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/rs.yaml @@ -15,7 +15,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/hpo_exp_scripts/uniqueness/personalized/rs_wrap.yaml b/scripts/hpo_exp_scripts/uniqueness/personalized/rs_wrap.yaml index 1bdf14d4a..201ca2625 100644 --- a/scripts/hpo_exp_scripts/uniqueness/personalized/rs_wrap.yaml +++ b/scripts/hpo_exp_scripts/uniqueness/personalized/rs_wrap.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/example_configs/femnist/avg/bo_gp.yaml b/scripts/hpo_exp_scripts/usability/femnist/bo_gp.yaml similarity index 91% rename from scripts/example_configs/femnist/avg/bo_gp.yaml rename to scripts/hpo_exp_scripts/usability/femnist/bo_gp.yaml index 35915c31f..af01e2cc5 100644 --- a/scripts/example_configs/femnist/avg/bo_gp.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/bo_gp.yaml @@ -15,7 +15,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/example_configs/femnist/avg/bo_gp_wrap.yaml b/scripts/hpo_exp_scripts/usability/femnist/bo_gp_wrap.yaml similarity index 92% rename from scripts/example_configs/femnist/avg/bo_gp_wrap.yaml rename to scripts/hpo_exp_scripts/usability/femnist/bo_gp_wrap.yaml index b7c9896e7..9d79e5db9 100644 --- a/scripts/example_configs/femnist/avg/bo_gp_wrap.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/bo_gp_wrap.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/example_configs/femnist/avg/bo_kde.yaml b/scripts/hpo_exp_scripts/usability/femnist/bo_kde.yaml similarity index 91% rename from scripts/example_configs/femnist/avg/bo_kde.yaml rename to scripts/hpo_exp_scripts/usability/femnist/bo_kde.yaml index e8096feeb..b91327f59 100644 --- a/scripts/example_configs/femnist/avg/bo_kde.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/bo_kde.yaml @@ -15,7 +15,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/example_configs/femnist/avg/bo_kde_wrap.yaml b/scripts/hpo_exp_scripts/usability/femnist/bo_kde_wrap.yaml similarity index 93% rename from scripts/example_configs/femnist/avg/bo_kde_wrap.yaml rename to scripts/hpo_exp_scripts/usability/femnist/bo_kde_wrap.yaml index 1ad4f0003..38d0d5099 100644 --- a/scripts/example_configs/femnist/avg/bo_kde_wrap.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/bo_kde_wrap.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/example_configs/femnist/avg/bo_rf.yaml b/scripts/hpo_exp_scripts/usability/femnist/bo_rf.yaml similarity index 91% rename from scripts/example_configs/femnist/avg/bo_rf.yaml rename to scripts/hpo_exp_scripts/usability/femnist/bo_rf.yaml index 1e8bca28e..145ad9785 100644 --- a/scripts/example_configs/femnist/avg/bo_rf.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/bo_rf.yaml @@ -15,7 +15,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/example_configs/femnist/avg/bo_rf_wrap.yaml b/scripts/hpo_exp_scripts/usability/femnist/bo_rf_wrap.yaml similarity index 92% rename from scripts/example_configs/femnist/avg/bo_rf_wrap.yaml rename to scripts/hpo_exp_scripts/usability/femnist/bo_rf_wrap.yaml index 7a46cb26c..31f05fdbd 100644 --- a/scripts/example_configs/femnist/avg/bo_rf_wrap.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/bo_rf_wrap.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/example_configs/femnist/avg/bohb.yaml b/scripts/hpo_exp_scripts/usability/femnist/bohb.yaml similarity index 91% rename from scripts/example_configs/femnist/avg/bohb.yaml rename to scripts/hpo_exp_scripts/usability/femnist/bohb.yaml index b60970464..c53d94ff9 100644 --- a/scripts/example_configs/femnist/avg/bohb.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/bohb.yaml @@ -15,7 +15,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/example_configs/femnist/avg/bohb_wrap.yaml b/scripts/hpo_exp_scripts/usability/femnist/bohb_wrap.yaml similarity index 92% rename from scripts/example_configs/femnist/avg/bohb_wrap.yaml rename to scripts/hpo_exp_scripts/usability/femnist/bohb_wrap.yaml index 83a74fb8c..dbae06851 100644 --- a/scripts/example_configs/femnist/avg/bohb_wrap.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/bohb_wrap.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/example_configs/femnist/avg/hb.yaml b/scripts/hpo_exp_scripts/usability/femnist/hb.yaml similarity index 91% rename from scripts/example_configs/femnist/avg/hb.yaml rename to scripts/hpo_exp_scripts/usability/femnist/hb.yaml index f48d9de93..6c6267cf9 100644 --- a/scripts/example_configs/femnist/avg/hb.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/hb.yaml @@ -15,7 +15,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/example_configs/femnist/avg/hb_wrap.yaml b/scripts/hpo_exp_scripts/usability/femnist/hb_wrap.yaml similarity index 93% rename from scripts/example_configs/femnist/avg/hb_wrap.yaml rename to scripts/hpo_exp_scripts/usability/femnist/hb_wrap.yaml index 1cfdeeca6..4a9cd60ee 100644 --- a/scripts/example_configs/femnist/avg/hb_wrap.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/hb_wrap.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/example_configs/femnist/avg/learn_from_scratch.yaml b/scripts/hpo_exp_scripts/usability/femnist/learn_from_scratch.yaml similarity index 89% rename from scripts/example_configs/femnist/avg/learn_from_scratch.yaml rename to scripts/hpo_exp_scripts/usability/femnist/learn_from_scratch.yaml index ac484e7dc..f6b322da5 100644 --- a/scripts/example_configs/femnist/avg/learn_from_scratch.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/learn_from_scratch.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/example_configs/femnist/avg/rs.yaml b/scripts/hpo_exp_scripts/usability/femnist/rs.yaml similarity index 92% rename from scripts/example_configs/femnist/avg/rs.yaml rename to scripts/hpo_exp_scripts/usability/femnist/rs.yaml index d6dd868a0..67616e04f 100644 --- a/scripts/example_configs/femnist/avg/rs.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/rs.yaml @@ -15,7 +15,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/example_configs/femnist/avg/rs_wrap.yaml b/scripts/hpo_exp_scripts/usability/femnist/rs_wrap.yaml similarity index 93% rename from scripts/example_configs/femnist/avg/rs_wrap.yaml rename to scripts/hpo_exp_scripts/usability/femnist/rs_wrap.yaml index 00f45d58e..70223b024 100644 --- a/scripts/example_configs/femnist/avg/rs_wrap.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/rs_wrap.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/example_configs/femnist/avg/rs_wrap_pfedex.sh b/scripts/hpo_exp_scripts/usability/femnist/rs_wrap_pfedex.sh similarity index 100% rename from scripts/example_configs/femnist/avg/rs_wrap_pfedex.sh rename to scripts/hpo_exp_scripts/usability/femnist/rs_wrap_pfedex.sh diff --git a/scripts/example_configs/femnist/avg/sha.yaml b/scripts/hpo_exp_scripts/usability/femnist/sha.yaml similarity index 91% rename from scripts/example_configs/femnist/avg/sha.yaml rename to scripts/hpo_exp_scripts/usability/femnist/sha.yaml index 0c9350fe0..cf6d5a387 100644 --- a/scripts/example_configs/femnist/avg/sha.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/sha.yaml @@ -15,7 +15,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/example_configs/femnist/avg/sha_wrap.yaml b/scripts/hpo_exp_scripts/usability/femnist/sha_wrap.yaml similarity index 92% rename from scripts/example_configs/femnist/avg/sha_wrap.yaml rename to scripts/hpo_exp_scripts/usability/femnist/sha_wrap.yaml index cf476e100..879c99a03 100644 --- a/scripts/example_configs/femnist/avg/sha_wrap.yaml +++ b/scripts/hpo_exp_scripts/usability/femnist/sha_wrap.yaml @@ -16,7 +16,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 16 model: diff --git a/scripts/example_configs/femnist/avg/ss.yaml b/scripts/hpo_exp_scripts/usability/femnist/ss.yaml similarity index 100% rename from scripts/example_configs/femnist/avg/ss.yaml rename to scripts/hpo_exp_scripts/usability/femnist/ss.yaml diff --git a/scripts/tianchi/submit.yaml b/scripts/tianchi/submit.yaml index 1ee4fbb61..9ecdbd23c 100644 --- a/scripts/tianchi/submit.yaml +++ b/scripts/tianchi/submit.yaml @@ -17,7 +17,7 @@ data: type: femnist splits: [0.6,0.2,0.2] subsample: 0.05 - transform: [['ToTensor'], ['Normalize', {'mean': [0.1307], 'std': [0.3081]}]] + transform: [['ToTensor'], ['Normalize', {'mean': [0.9637], 'std': [0.1592]}]] dataloader: batch_size: 10 model: