configurable thresholds for utxo merge policies

Author: adlai

bitcoin/secp256k1_transaction.py

@@ -433,20 +433,3 @@ def mktx(*args):
         txobj["outs"].append(outobj)
 
     return serialize(txobj)
-
-
-def select(unspent, value):
-    value = int(value)
-    high = [u for u in unspent if u["value"] >= value]
-    high.sort(key=lambda u: u["value"])
-    low = [u for u in unspent if u["value"] < value]
-    low.sort(key=lambda u: -u["value"])
-    if len(high):
-        return [high[0]]
-    i, tv = 0, 0
-    while tv < value and i < len(low):
-        tv += low[i]["value"]
-        i += 1
-    if tv < value:
-        raise Exception("Not enough funds")
-    return low[:i]

joinmarket/configure.py

@@ -121,11 +121,11 @@ def jm_single():
 confirm_timeout_hours = 6
 
 [POLICY]
-# for dust sweeping, try merge_algorithm = gradual
-# for more rapid dust sweeping, try merge_algorithm = greedy
-# for most rapid dust sweeping, try merge_algorithm = greediest
-# but don't forget to bump your miner fees!
-merge_algorithm = default
+# this knob is a list of ints, being each the amount of utxos in a mixdepth
+# sufficient for kicking in the next-mergiest utxo selection algorithm. the
+# default errs on the side of privacy; lower values leak more correlations.
+# it won't merge until the 42nd utxo, merges gradually until the 59th, etc
+merge_algorithm = [42, 59, 72]
 # For takers: the minimum number of makers you allow in a transaction
 # to complete, accounting for the fact that some makers might not be
 # responsive. Should be an integer >=2 for privacy, or set to 0 if you

joinmarket/support.py

@@ -98,6 +98,21 @@ def rand_weighted_choice(n, p_arr):
 def chunks(d, n):
     return [d[x:x + n] for x in xrange(0, len(d), n)]
 
+def select_default(unspent, value):
+    value = int(value)
+    high = [u for u in unspent if u["value"] >= value]
+    high.sort(key=lambda u: u["value"])
+    low = [u for u in unspent if u["value"] < value]
+    low.sort(key=lambda u: -u["value"])
+    if len(high):
+        return [high[0]]
+    i, tv = 0, 0
+    while tv < value and i < len(low):
+        tv += low[i]["value"]
+        i += 1
+    if tv < value:
+        raise Exception("Not enough funds")
+    return low[:i]
 
 def select_gradual(unspent, value):
     """
@@ -174,6 +189,21 @@ def select_greediest(unspent, value):
             end += 1
         return low[0:end]
 
+# ordered from most dusty (arguably, most private) to most mergiest (cheaper!)
+selectors = [select_default, select_gradual, select_greedy, select_greediest]
+
+def utxo_selector(configured_levels):
+    def select(unspent, value):
+        length = len(unspent)   # NB - counted only within each mixdepth
+        try:
+            for i in xrange(len(configured_levels)):
+                if length < configured_levels[i]:
+                    return selectors[i](unspent, value)
+            return selectors[len(configured_levels)](unspent, value)
+        except IndexError:
+            log.debug("Excess merge_algorithm levels. Configure fewer!")
+        return selectors[-1](unspent, value) # express operator greed
+    return select
 
 def calc_cj_fee(ordertype, cjfee, cj_amount):
     if ordertype == 'absoffer':

joinmarket/wallet.py

@@ -3,6 +3,7 @@
 import os
 import pprint
 import sys
+import ast
 from decimal import Decimal
 
 from ConfigParser import NoSectionError
@@ -13,8 +14,7 @@
 from joinmarket.blockchaininterface import BitcoinCoreInterface, RegtestBitcoinCoreInterface
 from joinmarket.configure import jm_single, get_network, get_p2pk_vbyte
 
-from joinmarket.support import get_log, select_gradual, select_greedy, \
-    select_greediest
+from joinmarket.support import get_log, utxo_selector
 
 log = get_log()
 
@@ -46,19 +46,27 @@ def __init__(self):
         #some consumer scripts don't use an unspent, this marks it
         #as specifically absent (rather than just empty).
         self.unspent = None
-        self.utxo_selector = btc.select  # default fallback: upstream
         try:
-            config = jm_single().config
-            if config.get("POLICY", "merge_algorithm") == "gradual":
-                self.utxo_selector = select_gradual
-            elif config.get("POLICY", "merge_algorithm") == "greedy":
-                self.utxo_selector = select_greedy
-            elif config.get("POLICY", "merge_algorithm") == "greediest":
-                self.utxo_selector = select_greediest
-            elif config.get("POLICY", "merge_algorithm") != "default":
-                raise Exception("Unknown merge algorithm")
+            policy = jm_single().config.get("POLICY", "merge_algorithm")
         except NoSectionError:
-            pass
+            policy = "default"  # maintain backwards compatibility!
+        if policy == "default":
+            self.merge_policy = [42] # well, almost (python lacks infinites)
+        elif policy == "gradual":
+            self.merge_policy = [60] # never goes beyond gradual
+        elif policy == "greedy":
+            self.merge_policy = [70, 70] # skip gradual, go greedy
+        elif policy == "greediest":
+            self.merge_policy = [80, 80, 80] # straight to greediest
+        else:
+            try:                # stop supporting word configs, someday...
+                self.merge_policy = ast.literal_eval(policy)
+                if ((type(self.merge_policy) is not list) or
+                    any(type(level) is not int for level in self.merge_policy)):
+                    raise Exception("Merge policy must be a list of ints")
+            except ValueError:
+                raise Exception("Unparseable merge policy: "+policy)
+        self.utxo_selector = utxo_selector(self.merge_policy)
 
     def get_key_from_addr(self, addr):
         return None

test/commontest.py

@@ -147,9 +147,8 @@ def make_wallets(n,
                                     'wallet': w}
         for j in range(5):
             for k in range(wallet_structures[i][j]):
-                deviation = sdev_amt * random.random()
-                amt = mean_amt - sdev_amt / 2.0 + deviation
-                if amt < 0: amt = 0.001
+                amt = random.gauss(mean_amt, sdev_amt)
+                if amt < 0.001: amt = 0.001
                 amt = float(Decimal(amt).quantize(Decimal(10)**-8))
                 jm_single().bc_interface.grab_coins(
                     wallets[i + start_index]['wallet'].get_external_addr(j),

test/regtest_joinmarket.cfg

@@ -21,11 +21,8 @@ socks5_port = 9150, 9150
 [LOGGING]
 console_log_level = DEBUG
 [POLICY]
-# for dust sweeping, try merge_algorithm = gradual
-# for more rapid dust sweeping, try merge_algorithm = greedy
-# for most rapid dust sweeping, try merge_algorithm = greediest
-# but don't forget to bump your miner fees!
-merge_algorithm = default
+# same as the default setting
+merge_algorithm = [42, 59, 72]
 # the fee estimate is based on a projection of how many satoshis
 # per kB are needed to get in one of the next N blocks, N set here
 # as the value of 'tx_fees'. This estimate can be extremely high

test/test_tx_creation.py

@@ -34,7 +34,7 @@
 
 @pytest.mark.parametrize(
     "nw, wallet_structures, mean_amt, sdev_amt, amount, pubs, k", [
-        (1, [[2, 1, 4, 0, 0]], 4, 1.4, 600000000, vpubs[1:4], 2),
+        (1, [[2, 1, 4, 0, 0]], 6, 1, 600000000, vpubs[1:4], 2),
         (1, [[3, 3, 0, 0, 3]], 4, 1.4, 100000000, vpubs[:4], 3),
     ])
 def test_create_p2sh_output_tx(setup_tx_creation, nw, wallet_structures,

test/test_wallets.py

@@ -350,6 +350,41 @@ def test_utxo_selection(setup_wallets, nw, wallet_structures, mean_amt,
                        "failed to select sufficient coins, total: " + \
                        str(total_selected) + ", should be: " + str(amount)
 
+# the following test mainly checks that the algo switching based on
+# number of utxos occurs correctly; it also provides some rough sanity
+# testing on the general behavior of the merge algorithms.
+# TODO: separate & deterministic tests for each merge algo!
+
+def test_merge_algo_switching(setup_wallets):
+    wallet = make_wallets(1, [[5, 50, 75, 0, 0]], 1, 0.72)[0]['wallet']
+    sync_wallet(wallet), sync_wallet(wallet)
+    all_utxos = wallet.get_utxos_by_mixdepth()
+    # test merge-avoiding with small utxo sets - select_default
+    utxos = sorted(map(lambda x:x['value'], all_utxos[0].values()))
+    for i in range(4):
+        amount = (utxos[i]+utxos[i+1])/2
+        selected = wallet.select_utxos(0, amount)
+        assert (1 == len(selected)), "Default selection misbehaved! " + \
+            "Selected " + str(selected) + " to reach sum " + str(amount) + \
+            " from utxos " + str(utxos) + " (should pick SINGLE utxo)"
+    # test merging with larger utxo sets - select_gradual
+    utxos = sorted(map(lambda x:x['value'], all_utxos[1].values()))
+    for i in range(49):
+        amount = utxos[i+1]+1
+        selected = wallet.select_utxos(1, amount)
+        assert (1 < len(selected)), "Default selection misbehaved! "+\
+            "Selected " + str(selected) + " to reach sum " + str(amount) + \
+            " from utxos " + str(utxos) + " (should pick MULTIPLE utxos)"
+    # TODO: test merging with intermediate sets - select_greedy
+    # test merging with even larger utxo sets - select_greediest
+    utxos = sorted(map(lambda x:x['value'], all_utxos[2].values()))
+    for i in range(74):
+        amount = sum(utxos[0:i+2])
+        selected = wallet.select_utxos(2, amount)
+        assert (i+2 <= len(selected)), "Default selection misbehaved! "+\
+            "Selected " + str(selected) + " to reach sum " + str(amount) + \
+            " from utxos " + str(utxos) + " (expected " + str(i+2) + ")"
+
 
 class TestWalletCreation(unittest.TestCase):