feat(config): env var override

Author: roshii

docs/USAGE.md

@@ -67,6 +67,10 @@ You should see the following files and folders for an initial setup:
 
 `joinmarket.cfg` is the main configuration file for Joinmarket and has a lot of settings, several of which you'll want to edit or at least examine.
 This will be discussed in several of the sections below.
+
+> **Environment variable overrides**  
+> Configuration values can be overridden using environment variables prefixed with `JM_`. Format: `JM_SECTION_KEY` (e.g., `JM_POLICY_TX_FEES`) or `JM_SECTION_SUBSECTION_KEY` for sections with subsections like `MESSAGING` (e.g., `JM_MESSAGING_ONION_TYPE`). When environment variables are present, they take precedence over the config file.
+
 The `wallets/` directory is where wallet files, extension (by default) of `.jmdat` are stored after you create them. They are encrypted and store important information; without them, it is possible to recover your coins with the seedphrase, but can be a hassle, so keep the file safe.
 The `logs/` directory contains a log file for each bot you run (Maker or Taker), with debug information. You'll rarely need to read these files unless you encounter a problem; deleting them regularly is recommended (and never dangerous). However there are other log files kept here, in particular one called `yigen-statement.csv` which records all transactions your Maker bot does over time. This can be useful for keeping track. Additionally, tumbles have a `TUMBLE.schedule` and `TUMBLE.log` file here which can be very useful; don't delete these.
 The `cmtdata/` directory stores technical information that you will not need to read.

src/jmclient/configure.py

@@ -1,5 +1,4 @@
 import atexit
-import io
 import logging
 import os
 import re
@@ -91,8 +90,10 @@ def jm_single() -> AttributeDict:
                     'POLICY': ['absurd_fee_per_kb', 'taker_utxo_retries',
                                'taker_utxo_age', 'taker_utxo_amtpercent']}
 
-_DEFAULT_INTEREST_RATE = "0.015"
+_ENV_VAR_PREFIX = "JM_"
+_SECTIONS_WITH_SUBSECTIONS = {"MESSAGING"}
 
+_DEFAULT_INTEREST_RATE = "0.015"
 _DEFAULT_BONDLESS_MAKERS_ALLOWANCE = "0.125"
 
 defaultconfig = \
@@ -673,9 +674,29 @@ def _remove_unwanted_default_settings(config: ConfigParser) -> None:
         if section.startswith('MESSAGING:'):
             config.remove_section(section)
 
-def load_program_config(config_path: str = "", bs: Optional[str] = None,
-                        plugin_services: List[JMPluginService] = []) -> None:
-    global_singleton.config.read_file(io.StringIO(defaultconfig))
+
+def override(config: ConfigParser | None) -> ConfigParser | None:
+    if not any(key.startswith(_ENV_VAR_PREFIX) for key in os.environ.keys()):
+        return config
+    if not config:
+        config = ConfigParser(strict=False)
+        config.read_string(defaultconfig)
+    for key, value in os.environ.items():
+        if key.startswith(_ENV_VAR_PREFIX):
+            key = key.removeprefix(_ENV_VAR_PREFIX)
+            section, key = key.split("_", 1)
+            if section in _SECTIONS_WITH_SUBSECTIONS:
+                sub, key = key.split("_", 1)
+                section = f"{section}:{sub.lower()}"
+            key = key.lower()
+            if not config.has_section(section):
+                config.add_section(section)
+            log.info(f"Overriding [{section}] {key}={value}")
+            config.set(section, key, value)
+    return config
+
+
+def _set_paths(config_path: str = "") -> None:
     if not config_path:
         config_path = lookup_appdata_folder(global_singleton.APPNAME)
     # we set the global home directory, but keep the config_path variable
@@ -692,29 +713,51 @@ def load_program_config(config_path: str = "", bs: Optional[str] = None,
     if not os.path.exists(os.path.join(global_singleton.datadir, "cmtdata")):
         os.makedirs(os.path.join(global_singleton.datadir, "cmtdata"))
     global_singleton.config_location = os.path.join(
-        global_singleton.datadir, global_singleton.config_location)
+        global_singleton.datadir, global_singleton.config_location
+    )
 
-    _remove_unwanted_default_settings(global_singleton.config)
+
+def read_config_file() -> ConfigParser | None:
+    config = ConfigParser(strict=False)
+    config.read_string(defaultconfig)
+    _remove_unwanted_default_settings(config)
     try:
-        loadedFiles = global_singleton.config.read(
-            [global_singleton.config_location])
+        loaded = config.read([global_singleton.config_location])
     except UnicodeDecodeError:
-        jmprint("Error loading `joinmarket.cfg`, invalid file format.",
-            "info")
+        jmprint("Error loading `joinmarket.cfg`, invalid file format.", "info")
+        sys.exit(EXIT_FAILURE)
+    return config if len(loaded) == 1 else None
+
+
+def write_config_file(config: str = defaultconfig) -> bool:
+    with open(global_singleton.config_location, "w") as configfile:
+        configfile.write(config)
+
+
+def load_program_config(
+    config_path: str = "",
+    bs: Optional[str] = None,
+    plugin_services: List[JMPluginService] = [],
+) -> None:
+    _set_paths(config_path)
+    config = read_config_file()
+    config = override(config)
+    # Create default config file if not found and no overrides
+    if not config:
+        write_config_file()
+        jmprint(
+            "Created a new `joinmarket.cfg`. Please review and adopt the "
+            "settings and restart joinmarket.",
+            "info",
+        )
         sys.exit(EXIT_FAILURE)
+    global_singleton.config = config
 
     # Hack required for bitcoin-rpc-no-history and probably others
     # (historicaly electrum); must be able to enforce a different blockchain
     # interface even in default/new load.
     if bs:
         global_singleton.config.set("BLOCKCHAIN", "blockchain_source", bs)
-    # Create default config file if not found
-    if len(loadedFiles) != 1:
-        with open(global_singleton.config_location, "w") as configfile:
-            configfile.write(defaultconfig)
-        jmprint("Created a new `joinmarket.cfg`. Please review and adopt the "
-              "settings and restart joinmarket.", "info")
-        sys.exit(EXIT_FAILURE)
 
     loglevel = global_singleton.config.get("LOGGING", "console_log_level")
     try:

test/jmclient/test_configure.py

@@ -1,8 +1,12 @@
 '''test configure module.'''
 
+import copy
+from configparser import ConfigParser
+
 import pytest
-from jmclient import load_test_config, jm_single
-from jmclient.configure import get_blockchain_interface_instance
+
+from jmclient import jm_single, load_test_config
+from jmclient.configure import get_blockchain_interface_instance, override
 
 pytestmark = pytest.mark.usefixtures("setup_regtest_bitcoind")
 
@@ -23,6 +27,8 @@ def test_load_config(tmpdir):
         load_test_config(config_path=str(tmpdir), bs="regtest")
     jm_single().config_location = "joinmarket.cfg"
     load_test_config()
+    ref = copy.deepcopy(jm_single().config)
+    assert override(jm_single().config) == ref
 
 
 def test_blockchain_sources():
@@ -35,3 +41,32 @@ def test_blockchain_sources():
         else:
             get_blockchain_interface_instance(jm_single().config)
     load_test_config()
+
+
+@pytest.fixture
+def overrides(monkeypatch):
+    overrides = {
+        "JM_BLOCKCHAIN_BLOCKCHAIN_SOURCE": "no-blockchain",
+        "JM_POLICY_TX_FEES": "12345678",
+        "JM_MESSAGING_ONION_TYPE": "lorem-ipsum",
+    }
+    for key, value in overrides.items():
+        monkeypatch.setenv(key, value)
+    return overrides
+
+
+def test_override(overrides):
+    config = ConfigParser()
+    override(config)
+    assert (
+        config.get("BLOCKCHAIN", "blockchain_source")
+        == overrides["JM_BLOCKCHAIN_BLOCKCHAIN_SOURCE"]
+    )
+    assert config.get("POLICY", "tx_fees") == overrides["JM_POLICY_TX_FEES"]
+    assert config.get("MESSAGING:onion", "type") == overrides["JM_MESSAGING_ONION_TYPE"]
+
+
+def test_load_program_config_overrides(overrides):
+    load_test_config()
+    assert jm_single().config.get("POLICY", "tx_fees") == overrides["JM_POLICY_TX_FEES"]
+    assert jm_single().config.get("MESSAGING:onion", "socks5_port") == "9050"