diff --git a/src/base/CMakeLists.txt b/src/base/CMakeLists.txt index 7c117ef4f1..41680c4a92 100644 --- a/src/base/CMakeLists.txt +++ b/src/base/CMakeLists.txt @@ -30,5 +30,5 @@ set(MY_PROJ_SRC "") dsn_add_static_library() -target_link_libraries(pegasus_base PUBLIC RocksDB::rocksdb sasl2 gssapi_krb5 krb5 dsn_replication_common) +target_link_libraries(pegasus_base PUBLIC dsn_replication_common RocksDB::rocksdb sasl2 gssapi_krb5 krb5 dsn_replication_common) target_include_directories(pegasus_base PUBLIC "$") diff --git a/src/common/duplication_common.cpp b/src/common/duplication_common.cpp index 4dc5323c65..116e580211 100644 --- a/src/common/duplication_common.cpp +++ b/src/common/duplication_common.cpp @@ -18,6 +18,7 @@ #include "duplication_common.h" #include +#include #include #include #include @@ -28,6 +29,7 @@ #include "nlohmann/detail/json_ref.hpp" #include "nlohmann/json_fwd.hpp" #include "utils/config_api.h" +#include "utils/configuration.h" #include "utils/error_code.h" #include "utils/fmt_logging.h" #include "utils/singleton.h" @@ -103,6 +105,59 @@ class duplication_group_registry : public utils::singleton &get_distinct_cluster_id_set() { return _distinct_cids; } + error_with reload_duplication_config(std::string config_file) + { + if (config_file.empty()) { + config_file = "config.ini"; + } + const char *config_file_cstr = config_file.c_str(); + + std::map new_group; + std::set new_distinct_cids; + dsn::configuration old_config; + + // reload default config.ini, user can point to another config file. update g_config here + if (!dsn_config_reload(config_file_cstr, nullptr, &old_config)) { + LOG_ERROR("Fail to reload config file {}", config_file_cstr); + return error_s::make( + ERR_OBJECT_NOT_FOUND, + " new `duplication-group` configured can not be read. Check your config.ini now"); + } + + int influented_clusters = 0; + + std::vector clusters; + dsn_config_get_all_keys("duplication-group", clusters); + // TODO(ninsmiracle): Add more illegal parameter conditional judgments + for (std::string &cluster : clusters) { + int64_t cluster_id = + dsn_config_get_value_int64("duplication-group", cluster.data(), 0, ""); + + // gns : do not dassert, just rolling it back and log error + if (cluster_id < 128 && cluster_id > 0) { + new_group.emplace(cluster, static_cast(cluster_id)); + new_distinct_cids.emplace(cluster_id); + } else { + LOG_ERROR( + "cluster_id({}) for {} should be in [1, 127]", cluster_id, cluster.data()); + // roll back cluster group and configuration + dsn_config_rollback(old_config); + return error_s::make(ERR_INVALID_PARAMETERS, + " new `duplication-group` configured invalid cluster id. " + "Check your config.ini now"); + } + } + + if (new_group.size() != _group.size()) { + influented_clusters = std::fabs(_group.size() - new_group.size()); + LOG_DEBUG("There are {} influented lines after reloading the config file", + influented_clusters); + } + + swap(new_group, _group); + swap(new_distinct_cids, _distinct_cids); + return influented_clusters; + } private: duplication_group_registry() @@ -161,6 +216,11 @@ class duplication_group_registry : public utils::singleton make_reloading_duplication_config(std::string &config_file) +{ + return internal::duplication_group_registry::instance().reload_duplication_config(config_file); +} + // TODO(wutao1): implement our C++ version of `TSimpleJSONProtocol` if there're // more cases for converting thrift to JSON static nlohmann::json duplication_entry_to_json(const duplication_entry &ent) diff --git a/src/common/duplication_common.h b/src/common/duplication_common.h index fe0ec067b4..b2e638db08 100644 --- a/src/common/duplication_common.h +++ b/src/common/duplication_common.h @@ -62,6 +62,8 @@ inline bool is_duplication_status_invalid(duplication_status::type status) /// The returned cluster id of get_duplication_cluster_id("wuhan-mi-srv-ad") is 3. extern error_with get_duplication_cluster_id(const std::string &cluster_name); +extern error_with make_reloading_duplication_config(std::string &config_file); + extern uint8_t get_current_dup_cluster_id_or_default(); extern uint8_t get_current_dup_cluster_id(); diff --git a/src/common/test/CMakeLists.txt b/src/common/test/CMakeLists.txt index fab0926249..ebed650cc2 100644 --- a/src/common/test/CMakeLists.txt +++ b/src/common/test/CMakeLists.txt @@ -40,6 +40,8 @@ set(MY_BOOST_LIBS Boost::system Boost::filesystem) set(MY_BINPLACES config-test.ini + err-config-test.ini + new-config-test.ini run.sh ) diff --git a/src/common/test/duplication_common_test.cpp b/src/common/test/duplication_common_test.cpp index 11690f4057..5eb071e97a 100644 --- a/src/common/test/duplication_common_test.cpp +++ b/src/common/test/duplication_common_test.cpp @@ -24,12 +24,17 @@ * THE SOFTWARE. */ -#include "common//duplication_common.h" +#include "common/duplication_common.h" #include +#include +#include +#include "common/replication_other_types.h" #include "gtest/gtest.h" +#include "runtime/rpc/rpc_host_port.h" #include "test_util/test_util.h" +#include "utils/config_api.h" #include "utils/error_code.h" #include "utils/flags.h" @@ -39,6 +44,11 @@ DSN_DECLARE_bool(dup_ignore_other_cluster_ids); namespace dsn { namespace replication { +std::string config_file = "config-test.ini"; +std::string unkown_file = "unknown.ini"; +std::string err_config_file = "err-config-test.ini"; +std::string new_config_file = "new-config-test.ini"; + TEST(duplication_common, get_duplication_cluster_id) { ASSERT_EQ(1, get_duplication_cluster_id("master-cluster").get_value()); @@ -82,5 +92,57 @@ TEST(duplication_common, dup_ignore_other_cluster_ids) } } +TEST(duplication_common, reload_config_file) +{ + ASSERT_EQ(make_reloading_duplication_config(unkown_file).get_error().code(), + ERR_OBJECT_NOT_FOUND); + ASSERT_EQ(make_reloading_duplication_config(err_config_file).get_error().code(), + ERR_INVALID_PARAMETERS); +} + +TEST(duplication_common, reload_get_duplication_cluster_id) +{ + make_reloading_duplication_config(config_file); + ASSERT_EQ(get_duplication_cluster_id("master-cluster").get_value(), 1); + ASSERT_EQ(get_duplication_cluster_id("slave-cluster").get_value(), 2); + ASSERT_EQ(get_duplication_cluster_id("strange-cluster").get_error().code(), + ERR_OBJECT_NOT_FOUND); + + make_reloading_duplication_config(new_config_file); + ASSERT_EQ(get_duplication_cluster_id("master-cluster").get_value(), 1); + ASSERT_EQ(get_duplication_cluster_id("slave-cluster").get_value(), 2); + ASSERT_EQ(get_duplication_cluster_id("strange-cluster").get_value(), 3); +} + +TEST(duplication_common, reload_get_meta_list) +{ + make_reloading_duplication_config(config_file); + replica_helper replica; + std::vector addr_vec; + // replica.load_meta_servers(addr_vec,"pegasus.clusters","strange-cluster"); + const char *strange_cluster_server_list = + dsn_config_get_value_string("pegasus.clusters", "strange-cluster", "", ""); + dsn::replication::replica_helper::parse_server_list(strange_cluster_server_list, addr_vec); + ASSERT_EQ(addr_vec.size(), 0); + addr_vec.clear(); + + make_reloading_duplication_config(new_config_file); + const char *new_strange_cluster_server_list = + dsn_config_get_value_string("pegasus.clusters", "strange-cluster", "", ""); + dsn::replication::replica_helper::parse_server_list(new_strange_cluster_server_list, addr_vec); + ASSERT_EQ(addr_vec.size(), 2); + + std::string addr0 = addr_vec[0].to_string(); + std::string addr1 = addr_vec[1].to_string(); + ASSERT_EQ(addr0, "localhost.test1:37001"); + ASSERT_EQ(addr1, "localhost.test2:37001"); + + addr_vec.clear(); + const char *unkonw_cluster_server_list = + dsn_config_get_value_string("pegasus.clusters", "unknow-cluster", "", ""); + dsn::replication::replica_helper::parse_server_list(unkonw_cluster_server_list, addr_vec); + ASSERT_EQ(addr_vec.size(), 0); +} + } // namespace replication } // namespace dsn diff --git a/src/common/test/err-config-test.ini b/src/common/test/err-config-test.ini new file mode 100644 index 0000000000..251b1c626c --- /dev/null +++ b/src/common/test/err-config-test.ini @@ -0,0 +1,107 @@ +; The MIT License (MIT) +; +; Copyright (c) 2015 Microsoft Corporation +; +; -=- Robust Distributed System Nucleus (rDSN) -=- +; +; Permission is hereby granted, free of charge, to any person obtaining a copy +; of this software and associated documentation files (the "Software"), to deal +; in the Software without restriction, including without limitation the rights +; to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +; copies of the Software, and to permit persons to whom the Software is +; furnished to do so, subject to the following conditions: +; +; The above copyright notice and this permission notice shall be included in +; all copies or substantial portions of the Software. +; +; THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +; IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +; FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +; AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +; LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +; OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +; THE SOFTWARE. + +[apps..default] +run = true +count = 1 +;network.client.RPC_CHANNEL_TCP = dsn::tools::sim_network_provider, 65536 +;network.client.RPC_CHANNEL_UDP = dsn::tools::sim_network_provider, 65536 +;network.server.0.RPC_CHANNEL_TCP = dsn::tools::sim_network_provider, 65536 + +[apps.replica] +type = replica +run = true +count = 1 +ports = 54321 +pools = THREAD_POOL_DEFAULT + +[core] +;tool = simulator +tool = nativerun + +;toollets = tracer, profiler +;fault_injector +pause_on_start = false +cli_local = false +cli_remote = false + +logging_start_level = LOG_LEVEL_DEBUG +logging_factory_name = dsn::tools::simple_logger + + +[tools.simple_logger] +fast_flush = true +short_header = false +stderr_start_level = LOG_LEVEL_WARNING + +[tools.simulator] +random_seed = 1465902258 + +[tools.screen_logger] +short_header = false + +[network] +; how many network threads for network library (used by asio) +io_service_worker_count = 2 + +; specification for each thread pool +[threadpool..default] +worker_count = 4 + +[threadpool.THREAD_POOL_DEFAULT] +name = default +partitioned = false +max_input_queue_length = 1024 +worker_priority = THREAD_xPRIORITY_NORMAL +worker_count = 2 + +[threadpool.THREAD_POOL_REPLICATION] +name = replica +partitioned = true +max_input_queue_length = 2560 +worker_priority = THREAD_xPRIORITY_NORMAL +worker_count = 3 + +[threadpool.THREAD_POOL_REPLICATION_LONG] +name = replica_long + +[task..default] +is_trace = true +is_profile = true +allow_inline = false +rpc_call_channel = RPC_CHANNEL_TCP +rpc_message_header_format = dsn +rpc_timeout_milliseconds = 5000 + +[replication] +cluster_name = master-cluster + +; if put strange-cluster as a invalid cluster id. So the config named err-config-test.ini +[duplication-group] +master-cluster = 1 +slave-cluster = 2 +strange-cluster = 6657 + +[pegasus.clusters] +strange-cluster = localhost.test1:37001,localhost.test2:37001 \ No newline at end of file diff --git a/src/common/test/new-config-test.ini b/src/common/test/new-config-test.ini new file mode 100644 index 0000000000..3b90ce2f6b --- /dev/null +++ b/src/common/test/new-config-test.ini @@ -0,0 +1,108 @@ +; The MIT License (MIT) +; +; Copyright (c) 2015 Microsoft Corporation +; +; -=- Robust Distributed System Nucleus (rDSN) -=- +; +; Permission is hereby granted, free of charge, to any person obtaining a copy +; of this software and associated documentation files (the "Software"), to deal +; in the Software without restriction, including without limitation the rights +; to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +; copies of the Software, and to permit persons to whom the Software is +; furnished to do so, subject to the following conditions: +; +; The above copyright notice and this permission notice shall be included in +; all copies or substantial portions of the Software. +; +; THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +; IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +; FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +; AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +; LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +; OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +; THE SOFTWARE. + +[apps..default] +run = true +count = 1 +;network.client.RPC_CHANNEL_TCP = dsn::tools::sim_network_provider, 65536 +;network.client.RPC_CHANNEL_UDP = dsn::tools::sim_network_provider, 65536 +;network.server.0.RPC_CHANNEL_TCP = dsn::tools::sim_network_provider, 65536 + +[apps.replica] +type = replica +run = true +count = 1 +ports = 54321 +pools = THREAD_POOL_DEFAULT + +[core] +;tool = simulator +tool = nativerun + +;toollets = tracer, profiler +;fault_injector +pause_on_start = false +cli_local = false +cli_remote = false + +logging_start_level = LOG_LEVEL_DEBUG +logging_factory_name = dsn::tools::simple_logger + + +[tools.simple_logger] +fast_flush = true +short_header = false +stderr_start_level = LOG_LEVEL_WARNING + +[tools.simulator] +random_seed = 1465902258 + +[tools.screen_logger] +short_header = false + +[network] +; how many network threads for network library (used by asio) +io_service_worker_count = 2 + +; specification for each thread pool +[threadpool..default] +worker_count = 4 + +[threadpool.THREAD_POOL_DEFAULT] +name = default +partitioned = false +max_input_queue_length = 1024 +worker_priority = THREAD_xPRIORITY_NORMAL +worker_count = 2 + +[threadpool.THREAD_POOL_REPLICATION] +name = replica +partitioned = true +max_input_queue_length = 2560 +worker_priority = THREAD_xPRIORITY_NORMAL +worker_count = 3 + +[threadpool.THREAD_POOL_REPLICATION_LONG] +name = replica_long + +[task..default] +is_trace = true +is_profile = true +allow_inline = false +rpc_call_channel = RPC_CHANNEL_TCP +rpc_message_header_format = dsn +rpc_timeout_milliseconds = 5000 + +[replication] +cluster_name = master-cluster + +; strange-cluster with a correct id +[duplication-group] +master-cluster = 1 +slave-cluster = 2 +strange-cluster = 3 + +; strange-cluster with a correct meta list +[pegasus.clusters] +strange-cluster = localhost.test1:37001,localhost.test2:37001 \ No newline at end of file diff --git a/src/runtime/CMakeLists.txt b/src/runtime/CMakeLists.txt index 6527ffb17f..3775bcf013 100644 --- a/src/runtime/CMakeLists.txt +++ b/src/runtime/CMakeLists.txt @@ -48,6 +48,6 @@ add_library(dsn_runtime STATIC tool_api.cpp tracer.cpp zlocks.cpp) -target_link_libraries(dsn_runtime PRIVATE dsn_security dsn_utils sasl2 gssapi_krb5 krb5) +target_link_libraries(dsn_runtime PRIVATE dsn_replication_common dsn_security dsn_utils sasl2 gssapi_krb5 krb5) define_file_basename_for_sources(dsn_runtime) install(TARGETS dsn_runtime DESTINATION "lib") diff --git a/src/runtime/service_api_c.cpp b/src/runtime/service_api_c.cpp index f4ed022344..47743aa89c 100644 --- a/src/runtime/service_api_c.cpp +++ b/src/runtime/service_api_c.cpp @@ -44,6 +44,7 @@ #include #endif +#include "common/duplication_common.h" #include "fmt/core.h" #include "fmt/format.h" #include "perf_counter/perf_counters.h" @@ -70,6 +71,7 @@ #include "utils/config_api.h" #include "utils/coredump.h" #include "utils/error_code.h" +#include "utils/errors.h" #include "utils/factory_store.h" #include "utils/filesystem.h" #include "utils/flags.h" @@ -118,6 +120,7 @@ static struct _all_info_ } dsn_all; std::unique_ptr dump_log_cmd; +std::unique_ptr reload_log_cmd; volatile int *dsn_task_queue_virtual_length_ptr(dsn::task_code code, int hash) { @@ -229,6 +232,7 @@ bool dsn_run_config(const char *config, bool is_server) [[noreturn]] void dsn_exit(int code) { dump_log_cmd.reset(); + reload_log_cmd.reset(); printf("dsn exit with code %d\n", code); fflush(stdout); @@ -585,6 +589,36 @@ bool run(const char *config_file, return oss.str(); }); + dump_log_cmd = dsn::command_manager::instance().register_single_command( + {"dup-config-reload"}, + "dup-config-reload - reload the new config file on every server", + "dup-config-reload [empty OR reload-this-config-file]. To dynamic update duplication " + "parameter " + "1.Put new config on every server.(Include replica server and meta server)" + "2.Use this remote command reload config to memory, and get duplication parameter to dup " + "object", + [](const std::vector &args) { + std::ostringstream oss; + std::string file_name = "config.ini"; + // choose another file to reload + if (args.size() > 0) { + file_name = args[0]; + } + + auto reload_influence_lines = + dsn::replication::make_reloading_duplication_config(file_name); + if (!reload_influence_lines.is_ok()) { + oss << "ERR_INVALID_PARAMETERS. Reload duplication config file, error:{" + << reload_influence_lines.get_error() << "}" << std::endl; + + return oss.str(); + } + + oss << "ERR_OK. Reload duplication config success, reload_influence_lines:{" + << std::to_string(reload_influence_lines.get_value()) << "}" << std::endl; + return oss.str(); + }); + // invoke customized init after apps are created dsn::tools::sys_init_after_app_created.execute(); diff --git a/src/utils/config_api.cpp b/src/utils/config_api.cpp index 6b5444760f..8c0b16d168 100644 --- a/src/utils/config_api.cpp +++ b/src/utils/config_api.cpp @@ -27,6 +27,7 @@ #include "utils/config_api.h" #include +#include #include "utils/configuration.h" @@ -39,6 +40,23 @@ bool dsn_config_load(const char *file, const char *arguments) void dsn_config_dump(std::ostream &os) { g_config.dump(os); } +bool dsn_config_reload(const char *file, + const char *arguments, + /*out*/ dsn::configuration *old_config) +{ + *old_config = g_config; + dsn::configuration temp_config; + if (!temp_config.load(file, arguments)) { + // TODO(ninsmiracle): Add some error log + return false; + } + + g_config = std::move(temp_config); + return true; +} + +void dsn_config_rollback(dsn::configuration &old_config) { g_config = std::move(old_config); } + const char *dsn_config_get_value_string(const char *section, const char *key, const char *default_value, diff --git a/src/utils/config_api.h b/src/utils/config_api.h index 0d8afd5d0c..937a4dcbc4 100644 --- a/src/utils/config_api.h +++ b/src/utils/config_api.h @@ -31,6 +31,10 @@ #include #include +namespace dsn { +class configuration; +} // namespace dsn + /// load a ini configuration file, and replace specific strings in file with arguments. /// /// the rules of replacement is as follows: @@ -63,6 +67,10 @@ /// the function is not thread safe. bool dsn_config_load(const char *file, const char *arguments); +bool dsn_config_reload(const char *file, const char *arguments, dsn::configuration *old_config); + +void dsn_config_rollback(dsn::configuration &old_config); + /// dump the global configuration void dsn_config_dump(std::ostream &os); diff --git a/src/utils/configuration.cpp b/src/utils/configuration.cpp index 449d1d4c16..1e50bd2092 100644 --- a/src/utils/configuration.cpp +++ b/src/utils/configuration.cpp @@ -40,7 +40,7 @@ namespace dsn { configuration::configuration() { _warning = false; } -configuration::~configuration() +void configuration::clear_configs() { for (auto §ion_kv : _configs) { auto §ion = section_kv.second; @@ -51,6 +51,19 @@ configuration::~configuration() _configs.clear(); } +void configuration::copy_configs(configuration &source_conf) +{ + source_conf._lock.lock(); + for (auto &[section_key, section] : source_conf._configs) { + for (auto &[key, value] : section) { + _configs[section_key][key] = new conf(*value); + } + } + source_conf._lock.unlock(); +} + +configuration::~configuration() { clear_configs(); } + // arguments: k1=v1;k2=v2;k3=v3; ... // e.g., // port = %port% diff --git a/src/utils/configuration.h b/src/utils/configuration.h index 533baf44b3..73be6cbb5b 100644 --- a/src/utils/configuration.h +++ b/src/utils/configuration.h @@ -35,6 +35,7 @@ #include #include #include +#include #include #include "string_conv.h" @@ -43,11 +44,62 @@ namespace dsn { class configuration { +private: + struct conf + { + std::string section; + std::string key; + std::string value; + int line; + + bool present; + std::string dsptr; + }; + + typedef std::map> config_map; + std::mutex _lock; + config_map _configs; + + std::string _file_name; + std::string _file_data; + bool _warning; + public: configuration(); ~configuration(); + void clear_configs(); + + void copy_configs(configuration &); + + configuration(configuration &conf) { copy_configs(conf); } + + configuration(const configuration &&conf) { _configs = std::move(conf._configs); } + + configuration &operator=(configuration &conf) + { + // deal with self assignment + if (this == &conf) + return *this; + _lock.lock(); + clear_configs(); + copy_configs(conf); + _lock.unlock(); + return *this; + } + + configuration &operator=(configuration &&conf) + { + if (this == &conf) + return *this; + _lock.lock(); + clear_configs(); + _configs = std::move(conf._configs); + _lock.unlock(); + return *this; + } + // arguments: k1=v1;k2=v2;k3=v3; ... // e.g., // port = %port% @@ -97,26 +149,6 @@ class configuration const char *default_value, const char **ov, const char *dsptr); - -private: - struct conf - { - std::string section; - std::string key; - std::string value; - int line; - - bool present; - std::string dsptr; - }; - - typedef std::map> config_map; - std::mutex _lock; - config_map _configs; - - std::string _file_name; - std::string _file_data; - bool _warning; }; template <>