From 2ada03eee9c46d6b5aee49b8b6dc7e37f9621752 Mon Sep 17 00:00:00 2001 From: tabVersion Date: Tue, 29 Sep 2020 11:37:44 +0800 Subject: [PATCH 01/16] integrate --- LearningMachine.py | 12 ++- ModelConf.py | 9 +++ config.yaml | 23 ++++++ exp.py | 35 +++++++++ lstm_base.json | 190 +++++++++++++++++++++++++++++++++++++++++++++ train.py | 27 +++++-- 6 files changed, 286 insertions(+), 10 deletions(-) create mode 100644 config.yaml create mode 100644 exp.py create mode 100644 lstm_base.json diff --git a/LearningMachine.py b/LearningMachine.py index 4b00240..7b49ed1 100644 --- a/LearningMachine.py +++ b/LearningMachine.py @@ -3,7 +3,6 @@ import torch import torch.nn as nn - import os import time import numpy as np @@ -12,6 +11,8 @@ import codecs import pickle as pkl +import nni + from utils.common_utils import dump_to_pkl, load_from_pkl, get_param_num, get_trainable_param_num, \ transfer_to_gpu, transform_params2tensors, get_layer_class, load_from_json, dump_to_json from utils.philly_utils import HDFSDirectTransferer, open_and_move, convert_to_tmppath, \ @@ -29,7 +30,7 @@ class LearningMachine(object): - def __init__(self, phase, conf, problem, vocab_info=None, initialize=True, use_gpu=False, **kwargs): + def __init__(self, phase, conf, problem, vocab_info=None, initialize=True, use_gpu=False, automl=False, **kwargs): if initialize is True: assert vocab_info is not None self.model = Model(conf, problem, vocab_info, use_gpu) @@ -54,6 +55,7 @@ def __init__(self, phase, conf, problem, vocab_info=None, initialize=True, use_g self.problem = problem self.phase = phase self.use_gpu = use_gpu + self.automl = automl # if it is a 2-class classification problem, figure out the real positive label # CAUTION: multi-class classification @@ -326,6 +328,8 @@ def train(self, optimizer, loss_fn): renew_flag = best_result != new_result best_result = new_result + # import ipdb; ipdb.set_trace() + if renew_flag and self.conf.test_data_path is not None: self.evaluate(test_data, test_length, test_target, self.conf.input_types, self.evaluator, loss_fn, pad_ids=None, phase="test", epoch=epoch) @@ -335,6 +339,8 @@ def train(self, optimizer, loss_fn): del data_batches, length_batches, target_batches lr_scheduler.step() epoch += 1 + if self.automl: + nni.report_final_result(float(best_result)) def test(self, loss_fn, test_data_path=None, predict_output_path=None): if test_data_path is None: @@ -622,6 +628,8 @@ def evaluate(self, data, length, target, input_types, evaluator, if phase == 'valid': cur_result = evaluator.get_first_metric_result() + if self.automl: + nni.report_intermediate_result(cur_result) if self.evaluator.compare(cur_result, cur_best_result) == 1: logging.info( 'Cur result %f is better than previous best result %s, renew the best model now...' % (cur_result, "%f" % cur_best_result if cur_best_result else "None")) diff --git a/ModelConf.py b/ModelConf.py index 9b170d0..23b1fb7 100644 --- a/ModelConf.py +++ b/ModelConf.py @@ -12,12 +12,16 @@ import logging import shutil +import nni + from losses.BaseLossConf import BaseLossConf #import traceback from settings import LanguageTypes, ProblemTypes, TaggingSchemes, SupportedMetrics, PredictionTypes, DefaultPredictionFields, ConstantStatic from utils.common_utils import log_set, prepare_dir, md5, load_from_json, dump_to_json from utils.exceptions import ConfigurationError import numpy as np +import torch +import random class ConstantStaticItems(ConstantStatic): @staticmethod @@ -174,6 +178,11 @@ def raise_configuration_error(self, key): def load_from_file(self, conf_path): # load file self.conf = load_from_json(conf_path, debug=False) + + if self.params.automl: + from exp import get_hyperparameters + self.conf = get_hyperparameters(self.conf) + self = self.Conf.load_data(self, {'Conf' : self.conf}, key_prefix_desc='Conf') self.language = self.language.lower() self.configurate_outputs() diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..88012b1 --- /dev/null +++ b/config.yaml @@ -0,0 +1,23 @@ +authorName: default +experimentName: lstm +trialConcurrency: 3 +maxExecDuration: 1000h +maxTrialNum: 500 +# 可选项: local, remote, pai, kubeflow +trainingServicePlatform: local +searchSpacePath: ./lstm_search_space.json +# 可选项: true, false, 默认值: false +useAnnotation: false +# 可选项: true, false, 默认值: false +multiThread: false +tuner: + builtinTunerName: TPE + classArgs: + optimize_mode: maximize + parallel_optimize: True + constant_liar_type: min +trial: + command: pip install -r requirements.txt && python train.py --conf_path lstm_base.json --automl + codeDir: . +localConfig: + useActiveGpu: true diff --git a/exp.py b/exp.py new file mode 100644 index 0000000..76bc844 --- /dev/null +++ b/exp.py @@ -0,0 +1,35 @@ +from nnicli import Experiment +import argparse +import os +import sys +import yaml +import argparse + + +def parser(): + parser = argparse.ArgumentParser() + parser.add_argument('--config_file', type=str, required=True, help='experiment config file') + parser.add_argument('--port', type=int, default=8080, help='show webUI on which port') + args = parser.parse_args() + return args + + +if __name__ == '__main__': + args = parse() + exp = Experiment() + # exp.stop_experiment() + exp.start_experiment(args.config_file, port=args.port) + + +def get_hyperparameters(conf): + parameters = nni.get_next_parameter() + conf["training_params"]["optimizer"]["params"]["lr"] = parameters["learning_rate"] + conf['architecture'][3]['conf']['dropout'] = parameters['LSTM_dropout_rate'] + conf['architecture'][4]['conf']['dropout'] = parameters['LSTM_dropout_rate'] + conf['training_params']['batch_size'] = parameters['batch_size'] + conf['architecture'][0]['conf']['word']['dropout'] = parameters['embedding_drop'] + conf['training_params']['lr_decay'] = parameters['decay'] + conf['loss']['losses'][0]['conf']['weight'] = [parameters['loss_weight'], 1 - parameters['loss_weight']] + conf['architecture'][2]['conf']['dropout'] = parameters['query_drop'] + conf['architecture'][3]['conf']['dropout'] = parameters['passage_drop'] + return conf diff --git a/lstm_base.json b/lstm_base.json new file mode 100644 index 0000000..0c1e2a9 --- /dev/null +++ b/lstm_base.json @@ -0,0 +1,190 @@ +{ + "license": "Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.", + "tool_version": "1.1.0", + "model_description": "This model is used for question answer matching task, and it achieved auc: 0.7854 in WikiQACorpus test set.", + "inputs": { + "use_cache": false, + "dataset_type": "classification", + "data_paths": { + "train_data_path": "/mnt/confignfs-data/NeuronBlocks_v-bzhang/qa_corpus/nb_qp_train_data.tsv", + "valid_data_path": "/mnt/confignfs-data/NeuronBlocks_v-bzhang/qa_corpus/nb_qp_dev_data.tsv", + "test_data_path": "/mnt/confignfs-data/NeuronBlocks_v-bzhang/qa_corpus/nb_qp_test_data.tsv", + "pre_trained_emb": "/mnt/confignfs-data/NeuronBlocks_v-bzhang/dataset/GloVe/glove.840B.300d.txt" + }, + "file_with_col_header": true, + "add_start_end_for_seq": true, + "file_header": { + "question_text": 0, + "passage_text": 1, + "label": 2 + }, + "model_inputs": { + "question": ["question_text"], + "passage": ["passage_text"] + }, + "target": ["label"] + }, + "outputs":{ + "save_base_dir": "models_wikiqa_matchatt/", + "model_name": "model.nb", + "train_log_name": "train.log", + "test_log_name": "test.log", + "predict_log_name": "predict.log", + "predict_fields": ["prediction"], + "predict_output_name": "predict.tsv" + }, + "training_params": { + "vocabulary": { + "min_word_frequency": 1 + }, + "optimizer": { + "name": "Adam", + "params": { + "lr": 0.002 + } + }, + "lr_decay": 1, + "minimum_lr": 0.0001, + "epoch_start_lr_decay": 3, + "steps_per_validation": 50, + "use_gpu": true, + "batch_size": 128, + "batch_num_to_show_results": 100, + "max_epoch": 10, + "valid_times_per_epoch": 5, + "max_lengths": { + "query": 50, + "passage": 200 + }, + "cpu_num_workers": 4 + }, + "architecture":[ + { + "layer": "Embedding", + "conf": { + "word": { + "cols": ["question_text", "passage_text"], + "dim": 300, + "fix_weight": true, + "dropout": 0 + } + } + }, + { + "layer_id": "query_dropout", + "layer": "Dropout", + "conf": { + "dropout": 0.1 + }, + "inputs": ["question"] + }, + { + "layer_id": "passage_dropout", + "layer": "Dropout", + "conf": { + "dropout": 0.1 + }, + "inputs": ["passage"] + }, + { + "layer_id": "query_1", + "layer": "BiLSTM", + "conf": { + "hidden_dim": 128, + "dropout": 0.1, + "num_layers": 2 + }, + "inputs": ["query_dropout"] + }, + { + "layer_id": "passage_1", + "layer": "BiLSTM", + "conf": { + "hidden_dim": 128, + "dropout": 0.1, + "num_layers": 2 + }, + "inputs": ["passage_dropout"] + }, + { + "layer_id": "query_matched", + "layer": "MatchAttention", + "conf": { + }, + "inputs": ["query_dropout", "passage_dropout"] + }, + { + "layer_id": "passage_matched", + "layer": "MatchAttention", + "conf": { + }, + "inputs": ["passage_dropout", "query_dropout"] + }, + { + "layer_id": "query_combined", + "layer": "Combination", + "conf": { + "operations": ["origin"] + }, + "inputs": ["query_1","query_matched"] + }, + { + "layer_id": "passage_combined", + "layer": "Combination", + "conf": { + "operations": ["origin"] + }, + "inputs": ["passage_1","passage_matched"] + }, + { + "layer_id": "query_linear_att", + "layer": "Pooling", + "conf": { + "pool_axis": 1, + "pool_type": "max" + }, + "inputs": ["query_combined"] + }, + { + "layer_id": "passage_linear_att", + "layer": "Pooling", + "conf": { + "pool_axis": 1, + "pool_type": "max" + }, + "inputs": ["passage_combined"] + }, + { + "layer_id": "comb", + "layer": "Combination", + "conf": { + "operations": ["origin"] + }, + "inputs": ["query_linear_att","passage_linear_att"] + }, + { + "output_layer_flag": true, + "layer_id": "output", + "layer": "Linear", + "conf": { + "hidden_dim": [128,2], + "activation": "PReLU", + "last_hidden_activation": false + }, + "inputs": ["comb"] + } + ], + "loss": { + "losses": [ + { + "type": "CrossEntropyLoss", + "conf": { + "weight": [0.4,0.6], + "size_average": true + }, + "inputs": ["output","label"] + } + ] + }, + "metrics": ["auc","accuracy"] +} diff --git a/train.py b/train.py index da92182..03690d8 100644 --- a/train.py +++ b/train.py @@ -1,6 +1,8 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT license. +import torch +import torch.nn as nn from settings import ProblemTypes, version, Setting as st import os @@ -11,8 +13,6 @@ import numpy as np import copy -import torch -import torch.nn as nn from ModelConf import ModelConf from problem import Problem from utils.common_utils import dump_to_pkl, load_from_pkl, load_from_json, dump_to_json, prepare_dir, md5 @@ -22,6 +22,9 @@ from LearningMachine import LearningMachine +logger = logging.getLogger() + + class Cache: def __init__(self): self.dictionary_invalid = True @@ -285,7 +288,7 @@ def main(params): if not conf.pretrained_model_path: vocab_info, initialize = get_vocab_info(conf, problem, emb_matrix), True - lm = LearningMachine('train', conf, problem, vocab_info=vocab_info, initialize=initialize, use_gpu=conf.use_gpu) + lm = LearningMachine('train', conf, problem, vocab_info=vocab_info, initialize=initialize, use_gpu=conf.use_gpu, automl=params.automl) if conf.pretrained_model_path: logging.info('Loading the pretrained model: %s...' % conf.pretrained_model_path) lm.load_model(conf.pretrained_model_path) @@ -355,6 +358,7 @@ def get_vocab_info(conf, problem, emb_matrix): if __name__ == "__main__": parser = argparse.ArgumentParser(description='Training') + parser.add_argument("--automl", type=bool, default=False, action='store_true', help="whether autoML will be used (rely on NNI)") parser.add_argument("--conf_path", type=str, help="configuration path") parser.add_argument("--train_data_path", type=str) parser.add_argument("--valid_data_path", type=str) @@ -362,18 +366,23 @@ def get_vocab_info(conf, problem, emb_matrix): parser.add_argument("--pretrained_emb_path", type=str) parser.add_argument("--pretrained_emb_type", type=str, default='glove', help='glove|word2vec|fasttext') parser.add_argument("--pretrained_emb_binary_or_text", type=str, default='text', help='text|binary') - parser.add_argument("--involve_all_words_in_pretrained_emb", type=bool, default=False, help='By default, only words that show up in the training data are involved.') + parser.add_argument("--involve_all_words_in_pretrained_emb", type=bool, default=False, + help='By default, only words that show up in the training data are involved.') parser.add_argument("--pretrained_model_path", type=str, help='load pretrained model, and then finetune it.') - parser.add_argument("--cache_dir", type=str, help='where stores the built problem.pkl where there are dictionaries like word2id, id2word. CAUTION: if there is a previous model, the dictionaries would be loaded from os.path.dir(previous_model_path)/.necessary_cache/problem.pkl') + parser.add_argument("--cache_dir", type=str, + help='where stores the built problem.pkl where there are dictionaries like word2id, id2word. CAUTION: if there is a previous model, the dictionaries would be loaded from os.path.dir(previous_model_path)/.necessary_cache/problem.pkl') parser.add_argument("--model_save_dir", type=str, help='where to store models') - parser.add_argument("--predict_output_path", type=str, help='specify another prediction output path, instead of conf[outputs][save_base_dir] + conf[outputs][predict_output_name] defined in configuration file') - parser.add_argument("--log_dir", type=str, help='If not specified, logs would be stored in conf_bilstmlast.json/outputs/save_base_dir') + parser.add_argument("--predict_output_path", type=str, + help='specify another prediction output path, instead of conf[outputs][save_base_dir] + conf[outputs][predict_output_name] defined in configuration file') + parser.add_argument("--log_dir", type=str, + help='If not specified, logs would be stored in conf_bilstmlast.json/outputs/save_base_dir') parser.add_argument("--make_cache_only", type=bool, default=False, help='make cache without training') parser.add_argument("--max_epoch", type=int, help='maximum number of epochs') parser.add_argument("--batch_size", type=int, help='batch_size of each gpu') parser.add_argument("--learning_rate", type=float, help='learning rate') parser.add_argument("--mode", type=str, default='normal', help='normal|philly') - parser.add_argument("--force", type=bool, default=False, help='Allow overwriting if some files or directories already exist.') + parser.add_argument("--force", type=bool, default=True, + help='Allow overwriting if some files or directories already exist.') parser.add_argument("--disable_log_file", type=bool, default=False, help='If True, disable log file') parser.add_argument("--debug", type=bool, default=False) @@ -381,6 +390,8 @@ def get_vocab_info(conf, problem, emb_matrix): # use for debug, remember delete # params.conf_path = 'configs_example/conf_debug_charemb.json' + logging.info("start once training") + assert params.conf_path, 'Please specify a configuration path via --conf_path' if params.pretrained_emb_path and not os.path.isabs(params.pretrained_emb_path): params.pretrained_emb_path = os.path.join(os.getcwd(), params.pretrained_emb_path) From c5d5ff3c2f7c6364aca43c799bf7de9daf5cd547 Mon Sep 17 00:00:00 2001 From: tabVersion Date: Tue, 29 Sep 2020 12:09:49 +0800 Subject: [PATCH 02/16] test pass --- config.yaml | 2 +- exp.py | 3 ++- lstm_base.json | 8 ++++---- lstm_search_space.json | 10 ++++++++++ train.py | 2 +- 5 files changed, 18 insertions(+), 7 deletions(-) create mode 100644 lstm_search_space.json diff --git a/config.yaml b/config.yaml index 88012b1..5e18e0a 100644 --- a/config.yaml +++ b/config.yaml @@ -1,6 +1,6 @@ authorName: default experimentName: lstm -trialConcurrency: 3 +trialConcurrency: 1 maxExecDuration: 1000h maxTrialNum: 500 # 可选项: local, remote, pai, kubeflow diff --git a/exp.py b/exp.py index 76bc844..db8f808 100644 --- a/exp.py +++ b/exp.py @@ -4,6 +4,7 @@ import sys import yaml import argparse +import nni def parser(): @@ -15,7 +16,7 @@ def parser(): if __name__ == '__main__': - args = parse() + args = parser() exp = Experiment() # exp.stop_experiment() exp.start_experiment(args.config_file, port=args.port) diff --git a/lstm_base.json b/lstm_base.json index 0c1e2a9..1fa2008 100644 --- a/lstm_base.json +++ b/lstm_base.json @@ -6,10 +6,10 @@ "use_cache": false, "dataset_type": "classification", "data_paths": { - "train_data_path": "/mnt/confignfs-data/NeuronBlocks_v-bzhang/qa_corpus/nb_qp_train_data.tsv", - "valid_data_path": "/mnt/confignfs-data/NeuronBlocks_v-bzhang/qa_corpus/nb_qp_dev_data.tsv", - "test_data_path": "/mnt/confignfs-data/NeuronBlocks_v-bzhang/qa_corpus/nb_qp_test_data.tsv", - "pre_trained_emb": "/mnt/confignfs-data/NeuronBlocks_v-bzhang/dataset/GloVe/glove.840B.300d.txt" + "train_data_path": "/data/data6/v-bzhang/neuralBlocks/qa_corpus/nb_qp_train_data.tsv", + "valid_data_path": "/data/data6/v-bzhang/neuralBlocks/qa_corpus/nb_qp_dev_data.tsv", + "test_data_path": "/data/data6/v-bzhang/neuralBlocks/qa_corpus/nb_qp_test_data.tsv", + "pre_trained_emb": "/data/data6/v-bzhang/neuralBlocks/glove.840B.300d.txt" }, "file_with_col_header": true, "add_start_end_for_seq": true, diff --git a/lstm_search_space.json b/lstm_search_space.json new file mode 100644 index 0000000..6b699c5 --- /dev/null +++ b/lstm_search_space.json @@ -0,0 +1,10 @@ +{ + "LSTM_dropout_rate": { "_type": "uniform", "_value": [0, 0.999] }, + "batch_size": { "_type": "choice", "_value": [32, 64, 128, 256] }, + "learning_rate": { "_type": "loguniform", "_value": [0.0001, 0.01] }, + "embedding_drop": { "_type": "uniform", "_value": [0, 0.999] }, + "passage_drop": { "_type": "uniform", "_value": [0, 0.999] }, + "query_drop": { "_type": "uniform", "_value": [0, 0.999] }, + "decay": { "_type": "uniform", "_value": [0.95, 1] }, + "loss_weight": { "_type": "uniform", "_value": [0.001, 0.999] } +} diff --git a/train.py b/train.py index 03690d8..dcf878c 100644 --- a/train.py +++ b/train.py @@ -358,7 +358,7 @@ def get_vocab_info(conf, problem, emb_matrix): if __name__ == "__main__": parser = argparse.ArgumentParser(description='Training') - parser.add_argument("--automl", type=bool, default=False, action='store_true', help="whether autoML will be used (rely on NNI)") + parser.add_argument("--automl", default=False, action='store_true', help="whether autoML will be used (rely on NNI)") parser.add_argument("--conf_path", type=str, help="configuration path") parser.add_argument("--train_data_path", type=str) parser.add_argument("--valid_data_path", type=str) From 203b120af92a1c8ce0ec36f287cee97f9556c7a8 Mon Sep 17 00:00:00 2001 From: tabVersion Date: Tue, 29 Sep 2020 12:16:25 +0800 Subject: [PATCH 03/16] fix --- LearningMachine.py | 4 ++-- train.py | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/LearningMachine.py b/LearningMachine.py index 7b49ed1..5239e1d 100644 --- a/LearningMachine.py +++ b/LearningMachine.py @@ -11,8 +11,6 @@ import codecs import pickle as pkl -import nni - from utils.common_utils import dump_to_pkl, load_from_pkl, get_param_num, get_trainable_param_num, \ transfer_to_gpu, transform_params2tensors, get_layer_class, load_from_json, dump_to_json from utils.philly_utils import HDFSDirectTransferer, open_and_move, convert_to_tmppath, \ @@ -56,6 +54,8 @@ def __init__(self, phase, conf, problem, vocab_info=None, initialize=True, use_g self.phase = phase self.use_gpu = use_gpu self.automl = automl + if self.automl: + import nni # if it is a 2-class classification problem, figure out the real positive label # CAUTION: multi-class classification diff --git a/train.py b/train.py index dcf878c..943638d 100644 --- a/train.py +++ b/train.py @@ -390,8 +390,6 @@ def get_vocab_info(conf, problem, emb_matrix): # use for debug, remember delete # params.conf_path = 'configs_example/conf_debug_charemb.json' - logging.info("start once training") - assert params.conf_path, 'Please specify a configuration path via --conf_path' if params.pretrained_emb_path and not os.path.isabs(params.pretrained_emb_path): params.pretrained_emb_path = os.path.join(os.getcwd(), params.pretrained_emb_path) From b86b3fa050faf2b7ca3325005c8a5a817d31cbb0 Mon Sep 17 00:00:00 2001 From: Tab Zhang Date: Tue, 29 Sep 2020 12:28:11 +0800 Subject: [PATCH 04/16] Update LearningMachine.py --- LearningMachine.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/LearningMachine.py b/LearningMachine.py index 5239e1d..f9c3a76 100644 --- a/LearningMachine.py +++ b/LearningMachine.py @@ -328,8 +328,6 @@ def train(self, optimizer, loss_fn): renew_flag = best_result != new_result best_result = new_result - # import ipdb; ipdb.set_trace() - if renew_flag and self.conf.test_data_path is not None: self.evaluate(test_data, test_length, test_target, self.conf.input_types, self.evaluator, loss_fn, pad_ids=None, phase="test", epoch=epoch) From c3606ff6c95c8f813470644ea109919bad915ab7 Mon Sep 17 00:00:00 2001 From: tabVersion Date: Wed, 30 Sep 2020 15:34:37 +0800 Subject: [PATCH 05/16] update --- LearningMachine.py | 4 ++-- ModelConf.py | 3 --- config.yaml | 1 + train.py | 4 +--- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/LearningMachine.py b/LearningMachine.py index f9c3a76..f327bd7 100644 --- a/LearningMachine.py +++ b/LearningMachine.py @@ -54,8 +54,6 @@ def __init__(self, phase, conf, problem, vocab_info=None, initialize=True, use_g self.phase = phase self.use_gpu = use_gpu self.automl = automl - if self.automl: - import nni # if it is a 2-class classification problem, figure out the real positive label # CAUTION: multi-class classification @@ -338,6 +336,7 @@ def train(self, optimizer, loss_fn): lr_scheduler.step() epoch += 1 if self.automl: + import nni nni.report_final_result(float(best_result)) def test(self, loss_fn, test_data_path=None, predict_output_path=None): @@ -627,6 +626,7 @@ def evaluate(self, data, length, target, input_types, evaluator, if phase == 'valid': cur_result = evaluator.get_first_metric_result() if self.automl: + import nni nni.report_intermediate_result(cur_result) if self.evaluator.compare(cur_result, cur_best_result) == 1: logging.info( diff --git a/ModelConf.py b/ModelConf.py index 23b1fb7..3ee7412 100644 --- a/ModelConf.py +++ b/ModelConf.py @@ -12,10 +12,7 @@ import logging import shutil -import nni - from losses.BaseLossConf import BaseLossConf -#import traceback from settings import LanguageTypes, ProblemTypes, TaggingSchemes, SupportedMetrics, PredictionTypes, DefaultPredictionFields, ConstantStatic from utils.common_utils import log_set, prepare_dir, md5, load_from_json, dump_to_json from utils.exceptions import ConfigurationError diff --git a/config.yaml b/config.yaml index 5e18e0a..a8d4d01 100644 --- a/config.yaml +++ b/config.yaml @@ -19,5 +19,6 @@ tuner: trial: command: pip install -r requirements.txt && python train.py --conf_path lstm_base.json --automl codeDir: . + gpuNum: 1 localConfig: useActiveGpu: true diff --git a/train.py b/train.py index 943638d..a1ad998 100644 --- a/train.py +++ b/train.py @@ -387,9 +387,7 @@ def get_vocab_info(conf, problem, emb_matrix): parser.add_argument("--debug", type=bool, default=False) params, _ = parser.parse_known_args() - # use for debug, remember delete - # params.conf_path = 'configs_example/conf_debug_charemb.json' - + assert params.conf_path, 'Please specify a configuration path via --conf_path' if params.pretrained_emb_path and not os.path.isabs(params.pretrained_emb_path): params.pretrained_emb_path = os.path.join(os.getcwd(), params.pretrained_emb_path) From 36f564a7527c5efed0e70503952cdf904fd33168 Mon Sep 17 00:00:00 2001 From: Tab Zhang Date: Mon, 5 Oct 2020 12:21:08 +0800 Subject: [PATCH 06/16] Update Tutorial.md --- Tutorial.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Tutorial.md b/Tutorial.md index ebe88dc..3fb73cc 100644 --- a/Tutorial.md +++ b/Tutorial.md @@ -26,6 +26,7 @@ * [Extra Feature Support](#extra-feature) * [Learning Rate Decay](#lr-decay) * [Fix Embedding Weight & Limit Vocabulary Size](#fix-embedding) +* [Auto Tuning Hyperparameters](#auto-ml) * [Frequently Asked Questions](#faq) ## Installation @@ -732,6 +733,14 @@ To solve the above problems, NeuronBlocks supports *fixing embedding weight* (em ***training_params/vocabulary/max_vocabulary***. [int, optional for training, default: 800,000] The max size of corpus vocabulary. If corpus vocabulary size is larger than *max_vocabulary*, it will be cut according to word frequency. ***training_params/vocabulary/max_building_lines***. [int, optional for training, default: 1,000,000] The max lines NB will read from every file to build vocabulary + +## Auto Tuning Hyperparameters + +This function integrates with [NNI](https://github.com/microsoft/nni) to try multiple set of hyperparameters generated by various tuning algorithms and run on different training environment. You can use `pip install nni` to install the dependency. + +Before launching an AutoML experiment, it is essential to specify the search space and tuning algorithm in `search_space.json` and `config.yaml` respectively and modify function `get_hyperparameters` in `exp.py`. Note that the function accepts the original architecture as its parameter and how the model uses these hyperparameters are defined here. More details can be found [here](https://nni.readthedocs.io/en/latest/Tutorial/QuickStart.html). + +After setting up, an experiment can be launched simply by `python3 exp.py --config_file CONFIG_FILE --port PORT`. ## Frequently Asked Questions From cf330f2f4865fb9e936f43fcf6c37e9db5e2ba62 Mon Sep 17 00:00:00 2001 From: Tab Zhang Date: Tue, 6 Oct 2020 11:06:13 +0800 Subject: [PATCH 07/16] Update Tutorial.md --- Tutorial.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tutorial.md b/Tutorial.md index 3fb73cc..699d623 100644 --- a/Tutorial.md +++ b/Tutorial.md @@ -736,7 +736,7 @@ To solve the above problems, NeuronBlocks supports *fixing embedding weight* (em ## Auto Tuning Hyperparameters -This function integrates with [NNI](https://github.com/microsoft/nni) to try multiple set of hyperparameters generated by various tuning algorithms and run on different training environment. You can use `pip install nni` to install the dependency. +This function integrates with [NNI](https://github.com/microsoft/nni) to try multiple sets of hyperparameters generated by various tuning algorithms and run on different training environments. You can use `pip install nni` to install the dependency. Before launching an AutoML experiment, it is essential to specify the search space and tuning algorithm in `search_space.json` and `config.yaml` respectively and modify function `get_hyperparameters` in `exp.py`. Note that the function accepts the original architecture as its parameter and how the model uses these hyperparameters are defined here. More details can be found [here](https://nni.readthedocs.io/en/latest/Tutorial/QuickStart.html). From ce5c2f4be97da14898639fa4cb3972066c0c96e6 Mon Sep 17 00:00:00 2001 From: Tab Zhang Date: Tue, 6 Oct 2020 14:40:03 +0800 Subject: [PATCH 08/16] Update Tutorial.md --- Tutorial.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Tutorial.md b/Tutorial.md index 699d623..bb92b27 100644 --- a/Tutorial.md +++ b/Tutorial.md @@ -738,8 +738,10 @@ To solve the above problems, NeuronBlocks supports *fixing embedding weight* (em This function integrates with [NNI](https://github.com/microsoft/nni) to try multiple sets of hyperparameters generated by various tuning algorithms and run on different training environments. You can use `pip install nni` to install the dependency. -Before launching an AutoML experiment, it is essential to specify the search space and tuning algorithm in `search_space.json` and `config.yaml` respectively and modify function `get_hyperparameters` in `exp.py`. Note that the function accepts the original architecture as its parameter and how the model uses these hyperparameters are defined here. More details can be found [here](https://nni.readthedocs.io/en/latest/Tutorial/QuickStart.html). +In an experiment, a set of hyperparameters is sampled from the search space, the space of hyperparameter configurations to search over, to get trial results. From these results, tuners can find better ranges for parameter values to fit a machine/deep learning model into different tasks/problems. +Before launching an AutoML experiment, it is essential to specify the search space and tuning algorithm in `search_space.json` and `config.yaml` respectively and modify function `get_hyperparameters` in `exp.py`. Note that the function accepts the original architecture as its parameter and how the model uses these hyperparameters are defined here. More details can be found [here](https://nni.readthedocs.io/en/latest/Tutorial/QuickStart.html). + After setting up, an experiment can be launched simply by `python3 exp.py --config_file CONFIG_FILE --port PORT`. ## Frequently Asked Questions From fd352b0159749e7a5a776cebc06c86ddbad5efe9 Mon Sep 17 00:00:00 2001 From: "Bohan Zhang (FA Talent)" Date: Fri, 9 Oct 2020 11:43:26 +0800 Subject: [PATCH 09/16] dir adjust --- config.yaml => autoML-demo/config.yaml | 0 exp.py => autoML-demo/exp.py | 72 +++++++++---------- lstm_base.json => autoML-demo/lstm_base.json | 0 .../lstm_search_space.json | 0 exp - Copy.py | 28 ++++++++ 5 files changed, 64 insertions(+), 36 deletions(-) rename config.yaml => autoML-demo/config.yaml (100%) rename exp.py => autoML-demo/exp.py (97%) rename lstm_base.json => autoML-demo/lstm_base.json (100%) rename lstm_search_space.json => autoML-demo/lstm_search_space.json (100%) create mode 100644 exp - Copy.py diff --git a/config.yaml b/autoML-demo/config.yaml similarity index 100% rename from config.yaml rename to autoML-demo/config.yaml diff --git a/exp.py b/autoML-demo/exp.py similarity index 97% rename from exp.py rename to autoML-demo/exp.py index db8f808..8174cd3 100644 --- a/exp.py +++ b/autoML-demo/exp.py @@ -1,36 +1,36 @@ -from nnicli import Experiment -import argparse -import os -import sys -import yaml -import argparse -import nni - - -def parser(): - parser = argparse.ArgumentParser() - parser.add_argument('--config_file', type=str, required=True, help='experiment config file') - parser.add_argument('--port', type=int, default=8080, help='show webUI on which port') - args = parser.parse_args() - return args - - -if __name__ == '__main__': - args = parser() - exp = Experiment() - # exp.stop_experiment() - exp.start_experiment(args.config_file, port=args.port) - - -def get_hyperparameters(conf): - parameters = nni.get_next_parameter() - conf["training_params"]["optimizer"]["params"]["lr"] = parameters["learning_rate"] - conf['architecture'][3]['conf']['dropout'] = parameters['LSTM_dropout_rate'] - conf['architecture'][4]['conf']['dropout'] = parameters['LSTM_dropout_rate'] - conf['training_params']['batch_size'] = parameters['batch_size'] - conf['architecture'][0]['conf']['word']['dropout'] = parameters['embedding_drop'] - conf['training_params']['lr_decay'] = parameters['decay'] - conf['loss']['losses'][0]['conf']['weight'] = [parameters['loss_weight'], 1 - parameters['loss_weight']] - conf['architecture'][2]['conf']['dropout'] = parameters['query_drop'] - conf['architecture'][3]['conf']['dropout'] = parameters['passage_drop'] - return conf +from nnicli import Experiment +import argparse +import os +import sys +import yaml +import argparse +import nni + + +def parser(): + parser = argparse.ArgumentParser() + parser.add_argument('--config_file', type=str, required=True, help='experiment config file') + parser.add_argument('--port', type=int, default=8080, help='show webUI on which port') + args = parser.parse_args() + return args + + +if __name__ == '__main__': + args = parser() + exp = Experiment() + # exp.stop_experiment() + exp.start_experiment(args.config_file, port=args.port) + + +def get_hyperparameters(conf): + parameters = nni.get_next_parameter() + conf["training_params"]["optimizer"]["params"]["lr"] = parameters["learning_rate"] + conf['architecture'][3]['conf']['dropout'] = parameters['LSTM_dropout_rate'] + conf['architecture'][4]['conf']['dropout'] = parameters['LSTM_dropout_rate'] + conf['training_params']['batch_size'] = parameters['batch_size'] + conf['architecture'][0]['conf']['word']['dropout'] = parameters['embedding_drop'] + conf['training_params']['lr_decay'] = parameters['decay'] + conf['loss']['losses'][0]['conf']['weight'] = [parameters['loss_weight'], 1 - parameters['loss_weight']] + conf['architecture'][2]['conf']['dropout'] = parameters['query_drop'] + conf['architecture'][3]['conf']['dropout'] = parameters['passage_drop'] + return conf diff --git a/lstm_base.json b/autoML-demo/lstm_base.json similarity index 100% rename from lstm_base.json rename to autoML-demo/lstm_base.json diff --git a/lstm_search_space.json b/autoML-demo/lstm_search_space.json similarity index 100% rename from lstm_search_space.json rename to autoML-demo/lstm_search_space.json diff --git a/exp - Copy.py b/exp - Copy.py new file mode 100644 index 0000000..4434d71 --- /dev/null +++ b/exp - Copy.py @@ -0,0 +1,28 @@ +from nnicli import Experiment +import argparse +import os +import sys +import yaml +import argparse +import nni + + +def parser(): + parser = argparse.ArgumentParser() + parser.add_argument('--config_file', type=str, required=True, help='experiment config file') + parser.add_argument('--port', type=int, default=8080, help='show webUI on which port') + args = parser.parse_args() + return args + + +if __name__ == '__main__': + args = parser() + exp = Experiment() + # exp.stop_experiment() + exp.start_experiment(args.config_file, port=args.port) + + +def get_hyperparameters(conf): + parameters = nni.get_next_parameter() + raise NotImplementedError + return conf From 00399aa5e4e0c6428525305d8fd6faf5db90c418 Mon Sep 17 00:00:00 2001 From: Tab Zhang Date: Fri, 9 Oct 2020 11:45:00 +0800 Subject: [PATCH 10/16] Rename exp - Copy.py to exp.py --- exp - Copy.py => exp.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename exp - Copy.py => exp.py (100%) diff --git a/exp - Copy.py b/exp.py similarity index 100% rename from exp - Copy.py rename to exp.py From 76fb1e6d7caa691264ca54ac07f5008b0a08c7a5 Mon Sep 17 00:00:00 2001 From: "Bohan Zhang (FA Talent)" Date: Fri, 9 Oct 2020 12:30:14 +0800 Subject: [PATCH 11/16] rename --- autoML-demo/config.yaml | 2 +- autoML-demo/{lstm_base.json => model.json} | 8 ++++---- autoML-demo/{lstm_search_space.json => search_space.json} | 0 3 files changed, 5 insertions(+), 5 deletions(-) rename autoML-demo/{lstm_base.json => model.json} (92%) rename autoML-demo/{lstm_search_space.json => search_space.json} (100%) diff --git a/autoML-demo/config.yaml b/autoML-demo/config.yaml index a8d4d01..b88606f 100644 --- a/autoML-demo/config.yaml +++ b/autoML-demo/config.yaml @@ -17,7 +17,7 @@ tuner: parallel_optimize: True constant_liar_type: min trial: - command: pip install -r requirements.txt && python train.py --conf_path lstm_base.json --automl + command: pip install -r requirements.txt && python train.py --conf_path model.json --automl codeDir: . gpuNum: 1 localConfig: diff --git a/autoML-demo/lstm_base.json b/autoML-demo/model.json similarity index 92% rename from autoML-demo/lstm_base.json rename to autoML-demo/model.json index 1fa2008..c436ef0 100644 --- a/autoML-demo/lstm_base.json +++ b/autoML-demo/model.json @@ -6,10 +6,10 @@ "use_cache": false, "dataset_type": "classification", "data_paths": { - "train_data_path": "/data/data6/v-bzhang/neuralBlocks/qa_corpus/nb_qp_train_data.tsv", - "valid_data_path": "/data/data6/v-bzhang/neuralBlocks/qa_corpus/nb_qp_dev_data.tsv", - "test_data_path": "/data/data6/v-bzhang/neuralBlocks/qa_corpus/nb_qp_test_data.tsv", - "pre_trained_emb": "/data/data6/v-bzhang/neuralBlocks/glove.840B.300d.txt" + "train_data_path": "./dataset/WikiQACorpus/WikiQA-train.tsv", + "valid_data_path": "./dataset/WikiQACorpus/WikiQA-train.tsv", + "test_data_path": "./dataset/WikiQACorpus/WikiQA-train.tsv", + "pre_trained_emb": "./dataset/Glove/glove.840B.300d.txt" }, "file_with_col_header": true, "add_start_end_for_seq": true, diff --git a/autoML-demo/lstm_search_space.json b/autoML-demo/search_space.json similarity index 100% rename from autoML-demo/lstm_search_space.json rename to autoML-demo/search_space.json From f01394cc1f73e7537f0f9161f51a0f359e2e09fa Mon Sep 17 00:00:00 2001 From: "Bohan Zhang (FA Talent)" Date: Fri, 9 Oct 2020 13:26:57 +0800 Subject: [PATCH 12/16] doc update --- Tutorial.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Tutorial.md b/Tutorial.md index bb92b27..f858a53 100644 --- a/Tutorial.md +++ b/Tutorial.md @@ -738,9 +738,15 @@ To solve the above problems, NeuronBlocks supports *fixing embedding weight* (em This function integrates with [NNI](https://github.com/microsoft/nni) to try multiple sets of hyperparameters generated by various tuning algorithms and run on different training environments. You can use `pip install nni` to install the dependency. -In an experiment, a set of hyperparameters is sampled from the search space, the space of hyperparameter configurations to search over, to get trial results. From these results, tuners can find better ranges for parameter values to fit a machine/deep learning model into different tasks/problems. +In an experiment, a set of hyperparameters is sampled from the search space, the space of hyperparameter configurations to search over, then a trial is conducted, and the result is collected. From these results, tuners can find parameter values within the search space that fit the task better. This process is known as Hyper Parameter Optimization (HPO). -Before launching an AutoML experiment, it is essential to specify the search space and tuning algorithm in `search_space.json` and `config.yaml` respectively and modify function `get_hyperparameters` in `exp.py`. Note that the function accepts the original architecture as its parameter and how the model uses these hyperparameters are defined here. More details can be found [here](https://nni.readthedocs.io/en/latest/Tutorial/QuickStart.html). +Before launching an AutoML experiment, it is essential to specify the search space and tuning algorithm in `search_space.json` and `config.yaml` respectively and modify function `get_hyperparameters` in `exp.py`. Note that the function accepts the original architecture as its parameter and how the model uses these hyperparameters are defined here. More details can be found [here](https://nni.readthedocs.io/en/latest/Tutorial/QuickStart.html). An example is available at `autoML-demo`. + +Here are some instructions about the files mentioned above. + +* `config.yaml`: In this file, basic experiment settings are specified, including experiment name, author name, which tuner to use, and how to start a trial. Users can also do some extra settings, like how many GPUs are available for one trial. +* `search_space.json`: Tuners are going to find hyperparameters within the range specified in this file. Users can set the prior distribution empirically for every hyperparameter searched to speed up the HPO process. +* `exp.py`: Function `get_hyperparameters` accepts the model from `model.json` as a parameter and get new hyperparameters from NNI. Some configurations in the model are replaced by the new hyperparameters manually in the function. After setting up, an experiment can be launched simply by `python3 exp.py --config_file CONFIG_FILE --port PORT`. From 7f3ca719c1b46a1a6d8728a2c7ee85b87fa8c491 Mon Sep 17 00:00:00 2001 From: "Bohan Zhang (FA Talent)" Date: Sat, 10 Oct 2020 10:11:08 +0800 Subject: [PATCH 13/16] doc update --- Tutorial.md | 4 ++-- Tutorial_zh_CN.md | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/Tutorial.md b/Tutorial.md index f858a53..24d5879 100644 --- a/Tutorial.md +++ b/Tutorial.md @@ -736,7 +736,7 @@ To solve the above problems, NeuronBlocks supports *fixing embedding weight* (em ## Auto Tuning Hyperparameters -This function integrates with [NNI](https://github.com/microsoft/nni) to try multiple sets of hyperparameters generated by various tuning algorithms and run on different training environments. You can use `pip install nni` to install the dependency. +This function integrates with [NNI](https://github.com/microsoft/nni) to try hyperparameters generated by various tuning algorithms and run experiments on different training environments. You can use `pip install nni` to install the dependency. In an experiment, a set of hyperparameters is sampled from the search space, the space of hyperparameter configurations to search over, then a trial is conducted, and the result is collected. From these results, tuners can find parameter values within the search space that fit the task better. This process is known as Hyper Parameter Optimization (HPO). @@ -746,7 +746,7 @@ Here are some instructions about the files mentioned above. * `config.yaml`: In this file, basic experiment settings are specified, including experiment name, author name, which tuner to use, and how to start a trial. Users can also do some extra settings, like how many GPUs are available for one trial. * `search_space.json`: Tuners are going to find hyperparameters within the range specified in this file. Users can set the prior distribution empirically for every hyperparameter searched to speed up the HPO process. -* `exp.py`: Function `get_hyperparameters` accepts the model from `model.json` as a parameter and get new hyperparameters from NNI. Some configurations in the model are replaced by the new hyperparameters manually in the function. +* `exp.py`: Function `get_hyperparameters` accepts the model from `model.json` as a parameter and get new hyperparameters from NNI. Some hyperparameters in the model are replaced by the new ones manually. After setting up, an experiment can be launched simply by `python3 exp.py --config_file CONFIG_FILE --port PORT`. diff --git a/Tutorial_zh_CN.md b/Tutorial_zh_CN.md index 501af09..f09110a 100644 --- a/Tutorial_zh_CN.md +++ b/Tutorial_zh_CN.md @@ -26,6 +26,7 @@ * [额外的feature](#extra-feature) * [学习率衰减](#lr-decay) * [固定embedding 和 词表大小设置](#fix-embedding) +* [自动超参数调整](#auto-ml) * [常见问题与答案](#faq) ## 安装 @@ -723,4 +724,29 @@ To solve the above problems, NeuronBlocks supports *fixing embedding weight* (em ***training_params/vocabulary/max_building_lines***. [int, optional for training, default: 1,000,000] The max lines NB will read from every file to build vocabulary +## 自动超参数调整 + +This function integrates with [NNI](https://github.com/microsoft/nni) to try multiple sets of hyperparameters generated by various tuning algorithms and run on different training environments. You can use `pip install nni` to install the dependency. + +此功能通过和 [NNI](https://github.com/microsoft/nni) 集成，以尝试通过多种自动调参算法生成超参数，并支持在多种平台上进行实验。您可以通过 `pip install nni` 来安装这个依赖。 + +In an experiment, a set of hyperparameters is sampled from the search space, the space of hyperparameter configurations to search over, then a trial is conducted, and the result is collected. From these results, tuners can find parameter values within the search space that fit the task better. This process is known as Hyper Parameter Optimization (HPO). + + +在实验中，首先从搜索空间中采样一组超参数，然后进行实验并收集结果。根据这些结果，自动调参算法可以在搜索空间中找到更适合当前任务的超参数值。这个过程称为超参数优化（HPO）。搜索空间指的是超参数的取值范围，自动调参算法将在这个空间内搜索合适的取值。 + +Before launching an AutoML experiment, it is essential to specify the search space and tuning algorithm in `search_space.json` and `config.yaml` respectively and modify function `get_hyperparameters` in `exp.py`. Note that the function accepts the original architecture as its parameter and how the model uses these hyperparameters are defined here. More details can be found [here](https://nni.readthedocs.io/en/latest/Tutorial/QuickStart.html). An example is available at `autoML-demo`. + +在启动自动机器学习实验之前，需要分别在 `search_space.json` 和 `config.yaml` 中指定搜索空间和调整算法，并在 `exp.py` 中修改函数 `get_hyperparameters`。请注意，该函数接受原始的模型作为其参数，并在这里定义了模型如何使用这些超参数。可以在[这里]（https://nni.readthedocs.io/en/latest/Tutorial/QuickStart.html）找到详细信息。在目录 `autoML-demo` 中可以找到一个已经配置好的例子。 + +Here are some instructions about the files mentioned above. + +以下是之前提到文件的一些说明。 + +* `config.yaml`：在此文件中，指定了基本的实验设置，包括实验名称，作者名称，要使用的调参算法以及开启实验的命令。用户还可以进行一些额外的设置，例如一次实验可以使用多少个GPU。 +* `search_space.json`：自动调参算法将在此文件中指定的范围内寻找超参数。用户可以根据经验为搜索的每个超参数设置先验分布来加快搜索。 +* `exp.py`：函数 `get_hyperparameters` 接受来自 `model.json` 的模型作为参数，并从 NNI 获取新的超参数。模型中的某些超参数需要被手动替换为新的。 + +设置完成后，只需通过 `python3 exp.py --config_file CONFIG_FILE --port PORT` 即可启动实验。 + ## 常见问题与答案 From e1e552bee85797dab5cfb6dccd6baf8e688f6d43 Mon Sep 17 00:00:00 2001 From: bytedance Date: Sat, 24 Oct 2020 11:58:54 +0800 Subject: [PATCH 14/16] automate reading tune parameters --- ModelConf.py | 16 +++++++++++++--- autoML-demo/exp.py | 14 -------------- autoML-demo/search_space.json | 14 ++++++-------- exp.py | 6 ------ 4 files changed, 19 insertions(+), 31 deletions(-) diff --git a/ModelConf.py b/ModelConf.py index 3ee7412..b1bb6f0 100644 --- a/ModelConf.py +++ b/ModelConf.py @@ -11,13 +11,13 @@ import torch import logging import shutil +from string import digits from losses.BaseLossConf import BaseLossConf from settings import LanguageTypes, ProblemTypes, TaggingSchemes, SupportedMetrics, PredictionTypes, DefaultPredictionFields, ConstantStatic from utils.common_utils import log_set, prepare_dir, md5, load_from_json, dump_to_json from utils.exceptions import ConfigurationError import numpy as np -import torch import random class ConstantStaticItems(ConstantStatic): @@ -177,8 +177,18 @@ def load_from_file(self, conf_path): self.conf = load_from_json(conf_path, debug=False) if self.params.automl: - from exp import get_hyperparameters - self.conf = get_hyperparameters(self.conf) + parameters = nni.get_next_parameter() + for para in parameters.keys(): + it = self.conf + for path in para.split('.'): + if path[0] in digits: + path = int(path) + try: + it = it[path] + except KeyError: + raise KeyError('Cannot access {} in parameter {}. Please check parameter names in search space file.' + .format(path, para)) + it = parameters[para] self = self.Conf.load_data(self, {'Conf' : self.conf}, key_prefix_desc='Conf') self.language = self.language.lower() diff --git a/autoML-demo/exp.py b/autoML-demo/exp.py index 8174cd3..6feb2f4 100644 --- a/autoML-demo/exp.py +++ b/autoML-demo/exp.py @@ -20,17 +20,3 @@ def parser(): exp = Experiment() # exp.stop_experiment() exp.start_experiment(args.config_file, port=args.port) - - -def get_hyperparameters(conf): - parameters = nni.get_next_parameter() - conf["training_params"]["optimizer"]["params"]["lr"] = parameters["learning_rate"] - conf['architecture'][3]['conf']['dropout'] = parameters['LSTM_dropout_rate'] - conf['architecture'][4]['conf']['dropout'] = parameters['LSTM_dropout_rate'] - conf['training_params']['batch_size'] = parameters['batch_size'] - conf['architecture'][0]['conf']['word']['dropout'] = parameters['embedding_drop'] - conf['training_params']['lr_decay'] = parameters['decay'] - conf['loss']['losses'][0]['conf']['weight'] = [parameters['loss_weight'], 1 - parameters['loss_weight']] - conf['architecture'][2]['conf']['dropout'] = parameters['query_drop'] - conf['architecture'][3]['conf']['dropout'] = parameters['passage_drop'] - return conf diff --git a/autoML-demo/search_space.json b/autoML-demo/search_space.json index 6b699c5..b1b16ef 100644 --- a/autoML-demo/search_space.json +++ b/autoML-demo/search_space.json @@ -1,10 +1,8 @@ { - "LSTM_dropout_rate": { "_type": "uniform", "_value": [0, 0.999] }, - "batch_size": { "_type": "choice", "_value": [32, 64, 128, 256] }, - "learning_rate": { "_type": "loguniform", "_value": [0.0001, 0.01] }, - "embedding_drop": { "_type": "uniform", "_value": [0, 0.999] }, - "passage_drop": { "_type": "uniform", "_value": [0, 0.999] }, - "query_drop": { "_type": "uniform", "_value": [0, 0.999] }, - "decay": { "_type": "uniform", "_value": [0.95, 1] }, - "loss_weight": { "_type": "uniform", "_value": [0.001, 0.999] } + "architecture.3.conf.dropout": { "_type": "uniform", "_value": [0, 0.999] }, + "training_params.batch_size": { "_type": "choice", "_value": [32, 64, 128, 256] }, + "training_params.optimizer.params.lr": { "_type": "loguniform", "_value": [0.0001, 0.01] }, + "architecture.0.conf.word.dropout": { "_type": "uniform", "_value": [0, 0.999] }, + "architecture.2.conf.dropout'": { "_type": "uniform", "_value": [0, 0.999] }, + "training_params.lr_decay": { "_type": "uniform", "_value": [0.95, 1] } } diff --git a/exp.py b/exp.py index 4434d71..6feb2f4 100644 --- a/exp.py +++ b/exp.py @@ -20,9 +20,3 @@ def parser(): exp = Experiment() # exp.stop_experiment() exp.start_experiment(args.config_file, port=args.port) - - -def get_hyperparameters(conf): - parameters = nni.get_next_parameter() - raise NotImplementedError - return conf From c0d1e6eabce447783579de3947b35f07e952d74d Mon Sep 17 00:00:00 2001 From: Tab Zhang Date: Mon, 26 Oct 2020 20:23:55 +0800 Subject: [PATCH 15/16] Update Tutorial.md --- Tutorial.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tutorial.md b/Tutorial.md index 24d5879..05f3ab4 100644 --- a/Tutorial.md +++ b/Tutorial.md @@ -745,7 +745,7 @@ Before launching an AutoML experiment, it is essential to specify the search spa Here are some instructions about the files mentioned above. * `config.yaml`: In this file, basic experiment settings are specified, including experiment name, author name, which tuner to use, and how to start a trial. Users can also do some extra settings, like how many GPUs are available for one trial. -* `search_space.json`: Tuners are going to find hyperparameters within the range specified in this file. Users can set the prior distribution empirically for every hyperparameter searched to speed up the HPO process. +* `search_space.json`: Tuners are going to find hyperparameters within the range specified in this file. Users can set the prior distribution empirically for every hyperparameter searched to speed up the HPO process. The names of tuning variables should follow the pattern to align with the architecture described in json file. Here is an example, if you want to tune `architecture['training_params']['batch_size']` automatically, the variable name in `search_space.json` should be `training_params.batch_size`. In a word, concatenate variable paths with dot to map from search space to architecture in json file. * `exp.py`: Function `get_hyperparameters` accepts the model from `model.json` as a parameter and get new hyperparameters from NNI. Some hyperparameters in the model are replaced by the new ones manually. After setting up, an experiment can be launched simply by `python3 exp.py --config_file CONFIG_FILE --port PORT`. From 166e5298c5883c03b75d6c2b923886b388810173 Mon Sep 17 00:00:00 2001 From: Tab Zhang Date: Mon, 26 Oct 2020 20:33:58 +0800 Subject: [PATCH 16/16] Update Tutorial_zh_CN.md --- Tutorial_zh_CN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tutorial_zh_CN.md b/Tutorial_zh_CN.md index f09110a..6950052 100644 --- a/Tutorial_zh_CN.md +++ b/Tutorial_zh_CN.md @@ -744,7 +744,7 @@ Here are some instructions about the files mentioned above. 以下是之前提到文件的一些说明。 * `config.yaml`：在此文件中，指定了基本的实验设置，包括实验名称，作者名称，要使用的调参算法以及开启实验的命令。用户还可以进行一些额外的设置，例如一次实验可以使用多少个GPU。 -* `search_space.json`：自动调参算法将在此文件中指定的范围内寻找超参数。用户可以根据经验为搜索的每个超参数设置先验分布来加快搜索。 +* `search_space.json`：自动调参算法将在此文件中指定的范围内寻找超参数。用户可以根据经验为搜索的每个超参数设置先验分布来加快搜索。search space文件里定义的变量通过一定的格式和描述结构的json文件中的变量进行关联。举个例子，如果需要调整 json 文件中的 `architecture['training_params']['batch_size']`，在 search_space 中的变量名应该是 `training_params.batch_size`。通过 `.` 连接访问希望调整的变量路径来关联描述模型结构的 json 文件和搜索空间中的变量。 * `exp.py`：函数 `get_hyperparameters` 接受来自 `model.json` 的模型作为参数，并从 NNI 获取新的超参数。模型中的某些超参数需要被手动替换为新的。 设置完成后，只需通过 `python3 exp.py --config_file CONFIG_FILE --port PORT` 即可启动实验。