Recursively merge configs from multiple sources

quetz/config.py

@@ -294,7 +294,7 @@ def init(self, path: str) -> None:
         if path:
             self.config.update(self._read_config(path))
 
-        self.config.update(self._get_environ_config())
+        self.config = recursively_merge_dictionaries(self.config, self._get_environ_config())
         self._trigger_update_config()
 
     def _trigger_update_config(self):
@@ -640,3 +640,28 @@ def get_plugin_manager(config=None) -> pluggy.PluginManager:
     else:
         pm.load_setuptools_entrypoints("quetz")
     return pm
+
+
+def recursively_merge_dictionaries(base_dict, other_dict):
+    """Recursively merge dictionaries. Overwrite duplicates with values from `other_dict`.
+
+    Parameters
+    ----------
+    base_dict : dict
+        The base dictionary to merge onto.
+    other_dict : dict
+        The dictionary containing prefered values.
+
+    Returns
+    -------
+    base_dict : dict
+        The dictionary containing the superset of `base_dict` and `other_dict` with values
+        of duplicate keys taken from `other_dict`.
+    """
+
+    for key, value in other_dict.items():
+        if key in base_dict and isinstance(base_dict[key], dict) and isinstance(value, dict):
+            base_dict[key] = recursively_merge_dictionaries(base_dict[key], value)
+        else:
+            base_dict[key] = value
+    return base_dict

quetz/testing/fixtures.py

@@ -164,10 +164,8 @@ def config_auth():
 
 
 @pytest.fixture
-def config_base(database_url, plugins, config_auth):
+def config_base(database_url, plugins):
     return f"""
-{config_auth}
-
 [sqlalchemy]
 database_url = "{database_url}"
 
@@ -180,6 +178,14 @@ def config_base(database_url, plugins, config_auth):
 """
 
 
+@pytest.fixture
+def config_base_with_auth(database_url, plugins, config_base, config_auth):
+    return f"""
+{config_auth}
+{config_base}
+"""
+
+
 @pytest.fixture
 def config_extra():
     return """

quetz/tests/test_config.py

@@ -70,13 +70,13 @@ def test_config_is_singleton(config):
     assert c_file is c_new
 
 
-def test_config_with_path(config_dir, config_base):
+def test_config_with_path(config_dir, config_base_with_auth):
     one_path = os.path.join(config_dir, "one_config.toml")
     other_path = os.path.join(config_dir, "other_config.toml")
     with open(one_path, "w") as fid:
-        fid.write("\n".join([config_base, "[users]\nadmins=['one']"]))
+        fid.write("\n".join([config_base_with_auth, "[users]\nadmins=['one']"]))
     with open(other_path, "w") as fid:
-        fid.write("\n".join([config_base, "[users]\nadmins=['other']"]))
+        fid.write("\n".join([config_base_with_auth, "[users]\nadmins=['other']"]))
 
     Config._instances = {}
 
@@ -160,3 +160,19 @@ def test_configure_logger(capsys):
     assert captured.err.count("second") == 1
     assert "my test" not in captured.err
     assert len(captured.err.splitlines()) == 1
+
+
+def test_config_from_multiple_sources(config_dir, config_base):
+    config_path = os.path.join(config_dir, "config.toml")
+    with open(config_path, "w") as fid:
+        fid.write("\n".join([config_base, "[github]\nclient_id='abc'"]))
+
+    Config._instances = {}
+
+    os.environ["QUETZ_GITHUB_CLIENT_SECRET"] = "abc"
+
+    c = Config(config_path)
+
+    assert c.configured_section("github")
+    assert c.github_client_id == "abc"
+    assert c.github_client_secret == "abc"