feat: tor managed services support

conftest.py

@@ -3,6 +3,7 @@
 import subprocess
 from shlex import split
 from time import sleep
+from twisted.internet import reactor
 from typing import Any, Tuple
 
 import pytest
@@ -172,3 +173,19 @@ def setup_regtest_bitcoind(pytestconfig):
     local_command(stop_cmd)
     # note, it is better to clean out ~/.bitcoin/regtest but too
     # dangerous to automate it here perhaps
+
+
+@pytest.fixture(autouse=True)
+def reset_reactor_state(request):
+    """Reset reactor _startedBefore flag after twisted.trial tests.
+
+    twisted.trial stops the reactor after each test, which marks it as
+    _startedBefore=True, preventing subsequent tests from running the reactor.
+    This fixture resets the flag to allow other tests to use the reactor.
+    """
+
+    def reset_flag():
+        if hasattr(reactor, "_startedBefore") and reactor._startedBefore:
+            if not reactor.running:
+                reactor._startedBefore = False
+    request.addfinalizer(reset_flag)

docs/onion-message-channels.md

@@ -59,6 +59,7 @@ onion_serving_port = 8080
 # but NOT TO BE USED by non-directory nodes (which is you, unless
 # you know otherwise!), as it will greatly degrade your privacy.
 # (note the default is no value, don't replace it with "").
+# Use tor-managed: prefix to use Tor-managed hidden services.
 hidden_service_dir =
 #
 # This is a comma separated list (comma can be omitted if only one item).
@@ -155,9 +156,11 @@ Add a non-empty `hidden_service_dir` entry to your `[MESSAGING:onion]` with a di
 
 The hostname for your onion service will not change and will be stored permanently in that directory.
 
-The point to understand is: Joinmarket's `jmbase.JMHiddenService` will, if configured with a non-empty `hidden_service_dir`
-field, actually start an *independent* instance of Tor specifically for serving this, under the current user.
-(our Tor interface library `txtorcon` needs read access to the Tor HS dir, so it's troublesome to do this another way).
+There are two ways to configure a persistent hidden service:
+
+1. **txtorcon-managed** (default when `hidden_service_dir` is set to a path): Joinmarket's `jmbase.JMHiddenService` will manage the hidden service via the Tor control port. This requires control port access and will start an *independent* instance of Tor specifically for serving this, under the current user. (our Tor interface library `txtorcon` needs read access to the Tor HS dir, so it's troublesome to do this another way).
+
+2. **Tor-managed** (when `hidden_service_dir` is prefixed with `tor-managed:`): Tor manages the hidden service via its `torrc` configuration file. JoinMarket reads the hostname from the `hostname` file in the specified directory. This mode does not require Tor control port access. See [Tor configuration documentation](./tor.md) for setup instructions.
 
 #### Question: How to configure the `directory-nodes` list in our `joinmarket.cfg` for this directory node bot?
 

docs/tor.md

@@ -83,3 +83,37 @@ sudo service tor start
 ```
 
 Once this is done, you should be able to start the yieldgenerator successfully.
+
+#### Tor-managed hidden services
+
+As an alternative to using the Tor control port, you can configure Tor to manage the hidden service directly via its configuration file (`torrc`). This approach is useful when:
+
+- You want Tor to fully manage the hidden service lifecycle
+- You don't want to grant control port access to JoinMarket
+- You're running Tor as a system service and prefer centralized configuration
+
+To use this mode:
+
+1. Configure the hidden service in Tor's `torrc` file (typically `/etc/tor/torrc`):
+
+   ```ini
+   HiddenServiceDir /var/lib/tor/joinmarket_hidden_service
+   HiddenServicePort 5222 127.0.0.1:8080
+   ```
+
+2. Set appropriate permissions
+3. Restart To
+4. Configure JoinMarket to use the Tor-managed service by setting `hidden_service_dir` in your `joinmarket.cfg`:
+
+   ```ini
+   hidden_service_dir = tor-managed:/var/lib/tor/joinmarket_hidden_service
+   ```
+
+   Note the `tor-managed:` prefix, which tells JoinMarket to read the hostname from the `hostname` file in that directory rather than managing the service via the control port.
+
+##### Important notes
+
+- The directory path in `hidden_service_dir` must match exactly what's configured in `torrc`
+- JoinMarket will read the hostname from the `hostname` file; make sure Tor has created it
+- No control port configuration is needed for this mode (though you may still need it for other features)
+- The hidden service directory must be readable by the user running JoinMarket (or the `hostname` file at minimum)

src/jmbase/twisted_utils.py

@@ -1,13 +1,17 @@
+import os
 
-from zope.interface import implementer
+import txtorcon
+from twisted.internet import defer, reactor
+from twisted.internet.endpoints import (
+    TCP4ClientEndpoint,
+    UNIXClientEndpoint,
+    serverFromString,
+)
 from twisted.internet.error import ReactorNotRunning
-from twisted.internet import reactor, defer
-from twisted.internet.endpoints import (TCP4ClientEndpoint,
-                        UNIXClientEndpoint, serverFromString)
 from twisted.web.client import Agent, BrowserLikePolicyForHTTPS
-import txtorcon
+from txtorcon import TorConfig, TorControlProtocol
 from txtorcon.web import tor_agent
-from txtorcon import TorControlProtocol, TorConfig
+from zope.interface import implementer
 
 _custom_stop_reactor_is_set = False
 custom_stop_reactor = None
@@ -170,48 +174,86 @@ def __init__(self, proto_factory_or_resource, info_callback,
         # an ephemeral HS on the global or pre-existing tor.
         self.hidden_service_dir = hidden_service_dir
 
+        self.tor_connection = None
+
     def start_tor(self):
         """ This function executes the workflow
         of starting the hidden service and returning its hostname
         """
-        self.info_callback("Attempting to start onion service on port: {} "
-                           "...".format(self.virtual_port))
+        self.info_callback(
+            f"Attempting to start onion service on port: {self.virtual_port} ..."
+        )
+
+        # Check if using Tor-managed mode (via torrc, not control port)
+        if self.hidden_service_dir.startswith("tor-managed:"):
+            self.start_tor_managed_onion()
+            return
+
+        # Ephemeral or txtorcon-managed hidden service (via control port)
+        if str(self.tor_control_host).startswith("unix:"):
+            control_endpoint = UNIXClientEndpoint(reactor, self.tor_control_host[5:])
+        else:
+            control_endpoint = TCP4ClientEndpoint(
+                reactor, self.tor_control_host, self.tor_control_port
+            )
+        d = txtorcon.connect(reactor, control_endpoint)
+
         if self.hidden_service_dir == "":
-            if str(self.tor_control_host).startswith('unix:'):
-                control_endpoint = UNIXClientEndpoint(reactor,
-                                        self.tor_control_host[5:])
-            else:
-                control_endpoint = TCP4ClientEndpoint(reactor,
-                                self.tor_control_host, self.tor_control_port)
-            d = txtorcon.connect(reactor, control_endpoint)
+            # Ephemeral hidden service (no persistence)
             d.addCallback(self.create_onion_ep)
             d.addErrback(self.setup_failed)
-            # TODO: add errbacks to the next two calls in
-            # the chain:
             d.addCallback(self.onion_listen)
             d.addCallback(self.print_host)
         else:
-            ep = "onion:" + str(self.virtual_port) + ":localPort="
-            ep += str(self.serving_port)
-            # endpoints.TCPHiddenServiceEndpoint creates version 2 by
-            # default for backwards compat (err, txtorcon needs to update that ...)
-            ep += ":version=3"
-            ep += ":hiddenServiceDir="+self.hidden_service_dir
-            onion_endpoint = serverFromString(reactor, ep)
-            d = onion_endpoint.listen(self.proto_factory)
+            # txtorcon-managed filesystem hidden service
+            d.addCallback(self.create_filesystem_onion_ep)
+            d.addErrback(self.setup_failed)
             d.addCallback(self.print_host_filesystem)
 
-
     def setup_failed(self, arg):
         # Note that actions based on this failure are deferred to callers:
         self.error_callback("Setup failed: " + str(arg))
 
     def create_onion_ep(self, t):
         self.tor_connection = t
-        portmap_string = config_to_hs_ports(self.virtual_port,
-                                self.serving_host, self.serving_port)
+        portmap_string = config_to_hs_ports(
+            self.virtual_port, self.serving_host, self.serving_port
+        )
         return t.create_onion_service(
-            ports=[portmap_string], private_key=txtorcon.DISCARD)
+            ports=[portmap_string], private_key=txtorcon.DISCARD
+        )
+
+    def create_filesystem_onion_ep(self, t):
+        """Create a persistent hidden service using txtorcon's filesystem support.
+        Requires local Tor control port access.
+        """
+        self.tor_connection = t
+        ep = "onion:" + str(self.virtual_port) + ":localPort="
+        ep += str(self.serving_port)
+        ep += ":version=3"
+        ep += ":hiddenServiceDir=" + self.hidden_service_dir
+        onion_endpoint = serverFromString(reactor, ep)
+        return onion_endpoint.listen(self.proto_factory)
+
+    def start_tor_managed_onion(self) -> None:
+        """
+        For Tor-managed hidden services: read hostname, start listening.
+        No control port connection needed.
+        """
+        hs_dir = self.hidden_service_dir.removeprefix("tor-managed:")
+        hostname_file = os.path.join(hs_dir, "hostname")
+
+        if not os.path.exists(hostname_file):
+            self.error_callback(f"Hostname file {hostname_file} does not exist")
+            return
+
+        try:
+            with open(hostname_file, "r") as f:
+                hostname = f.read().strip()
+            self.info_callback(f"Using Tor-managed hidden service: {hostname}")
+            self.onion_hostname_callback(hostname)
+        except Exception as e:
+            self.error_callback(f"Failed to read {hostname_file}: {e}")
 
     def onion_listen(self, onion):
         # 'onion' arg is the created EphemeralOnionService object;
@@ -240,11 +282,13 @@ def print_host_filesystem(self, port):
         self.onion_hostname_callback(self.onion.hostname)
 
     def shutdown(self):
-        self.tor_connection.protocol.transport.loseConnection()
+        if self.tor_connection:
+            self.tor_connection.protocol.transport.loseConnection()
         self.info_callback("Hidden service shutdown complete")
         if self.shutdown_callback:
             self.shutdown_callback()
 
+
 class JMHTTPResource(Resource):
     """ Object acting as HTTP serving resource
     """

src/jmclient/configure.py

@@ -172,6 +172,7 @@ def jm_single() -> AttributeDict:
 # but NOT TO BE USED by non-directory nodes (which is you, unless
 # you know otherwise!), as it will greatly degrade your privacy.
 # (note the default is no value, don't replace it with "").
+# Use tor-managed: prefix to use Tor-managed hidden services.
 hidden_service_dir =
 #
 # This is a comma separated list (comma can be omitted if only one item).

src/jmdaemon/onionmc.py

@@ -8,7 +8,7 @@
 from twisted.internet import reactor, task, protocol
 from twisted.protocols import basic
 from twisted.application.internet import ClientService
-from twisted.internet.endpoints import TCP4ClientEndpoint
+from twisted.internet.endpoints import serverFromString, TCP4ClientEndpoint
 from twisted.internet.address import IPv4Address, IPv6Address
 from txtorcon.socks import (TorSocksEndpoint, HostUnreachableError,
                             SocksError, GeneralServerFailureError)
@@ -697,9 +697,12 @@ def __init__(self,
                 # it'll fire the `setup_error_callback`.
                 self.hs.start_tor()
 
-                # This will serve as our unique identifier, indicating
-                # that we are ready to communicate (in both directions) over Tor.
-                self.onion_hostname = None
+                # For tor-managed services, the hostname is set synchronously by start_tor()
+                # For ephemeral services, we need to wait for the callback
+                if not self.hidden_service_dir.startswith("tor-managed:"):
+                    # This will serve as our unique identifier, indicating
+                    # that we are ready to communicate (in both directions) over Tor.
+                    self.onion_hostname = None
         else:
             # dummy 'hostname' to indicate we can start running immediately:
             self.onion_hostname = NOT_SERVING_ONION_HOSTNAME
@@ -884,7 +887,7 @@ def connect_to_directories(self) -> None:
         if self.genesis_node:
             # we are a directory and we have no directory peers;
             # just start.
-            self.on_welcome(self)
+            self._start_listener()
             return
         # the remaining code is only executed by non-directories:
         for p in self.peers:
@@ -901,6 +904,13 @@ def connect_to_directories(self) -> None:
             self.wait_for_directories)
         self.wait_for_directories_loop.start(2.0)
 
+    def _start_listener(self) -> None:
+        serverstring = f"tcp:{self.onion_serving_port}:interface={self.onion_serving_host}"
+        onion_endpoint = serverFromString(reactor, serverstring)
+        d = onion_endpoint.listen(self.proto_factory)
+        d.addCallback(self.on_welcome)
+        d.addErrback(lambda f: self.setup_error_callback(f"Listen failed: {f}"))
+
     def handshake_as_client(self, peer: OnionPeer) -> None:
         assert peer.status() == PEER_STATUS_CONNECTED
         if self.self_as_peer.directory:
@@ -1461,7 +1471,8 @@ def wait_for_directories(self) -> None:
         # Note that even if the preceding (max) 50 seconds failed to
         # connect all our configured dps, we will keep trying and they
         # can still be used.
-        if not self.on_welcome_sent:
+        # For genesis nodes, on_welcome is called after the listener starts
+        if not self.on_welcome_sent and not self.genesis_node:
             self.on_welcome(self)
             self.on_welcome_sent = True
             self.wait_for_directories_loop.stop()

test/jmbase/test_twisted_utils.py

@@ -0,0 +1,65 @@
+from unittest.mock import Mock, patch
+
+import pytest
+
+from jmbase.twisted_utils import JMHiddenService
+
+
+def mock_hs(hidden_service_dir: str = "") -> JMHiddenService:
+    return JMHiddenService(
+        Mock(),
+        Mock(),
+        Mock(),
+        Mock(),
+        "127.0.0.1",
+        9051,
+        "127.0.0.1",
+        8080,
+        80,
+        None,
+        hidden_service_dir,
+    )
+
+
+class TestTorManagedHiddenService:
+    @pytest.mark.parametrize(
+        "hidden_service_dir,expect_managed,expect_connect",
+        [
+            ("tor-managed:/path/to/dir", True, False),
+            ("/normal/path", False, True),
+        ],
+    )
+    def test_hidden_service_dir_detection(
+        self, hidden_service_dir, expect_managed, expect_connect
+    ):
+        with (
+            patch.object(JMHiddenService, "start_tor_managed_onion") as mock_managed,
+            patch("jmbase.twisted_utils.txtorcon.connect") as mock_connect,
+        ):
+            hs = mock_hs(hidden_service_dir)
+
+            hs.start_tor()
+
+            if expect_managed:
+                mock_managed.assert_called_once()
+                mock_connect.assert_not_called()
+            else:
+                mock_managed.assert_not_called()
+                mock_connect.assert_called_once()
+
+    def test_ephemeral_service_creation(self):
+        with patch("jmbase.twisted_utils.txtorcon") as mock_txtorcon:
+            mock_t = Mock()
+            mock_t.create_onion_service.return_value = Mock()
+
+            hs = mock_hs()
+            hs.tor_connection = mock_t
+            hs.virtual_port = 80
+            hs.serving_host = "127.0.0.1"
+            hs.serving_port = 8080
+
+            hs.create_onion_ep(mock_t)
+
+            mock_t.create_onion_service.assert_called_once_with(
+                ports=["80 127.0.0.1:8080"], private_key=mock_txtorcon.DISCARD
+            )