diff --git a/bitcoin/secp256k1_transaction.py b/bitcoin/secp256k1_transaction.py index 6658f622..bfb3c4dc 100644 --- a/bitcoin/secp256k1_transaction.py +++ b/bitcoin/secp256k1_transaction.py @@ -433,20 +433,3 @@ def mktx(*args): txobj["outs"].append(outobj) return serialize(txobj) - - -def select(unspent, value): - value = int(value) - high = [u for u in unspent if u["value"] >= value] - high.sort(key=lambda u: u["value"]) - low = [u for u in unspent if u["value"] < value] - low.sort(key=lambda u: -u["value"]) - if len(high): - return [high[0]] - i, tv = 0, 0 - while tv < value and i < len(low): - tv += low[i]["value"] - i += 1 - if tv < value: - raise Exception("Not enough funds") - return low[:i] diff --git a/joinmarket/configure.py b/joinmarket/configure.py index bcb776ea..a3f2ffa3 100644 --- a/joinmarket/configure.py +++ b/joinmarket/configure.py @@ -125,11 +125,11 @@ def jm_single(): confirm_timeout_hours = 6 [POLICY] -# for dust sweeping, try merge_algorithm = gradual -# for more rapid dust sweeping, try merge_algorithm = greedy -# for most rapid dust sweeping, try merge_algorithm = greediest -# but don't forget to bump your miner fees! -merge_algorithm = default +# this knob is a list of ints, being each the amount of utxos in a mixdepth +# sufficient for kicking in the next-mergiest utxo selection algorithm. the +# default errs on the side of privacy; lower values leak more correlations. +# it won't merge until the 42nd utxo, merges gradually until the 59th, etc +merge_algorithm = [42, 59, 72] # For takers: the minimum number of makers you allow in a transaction # to complete, accounting for the fact that some makers might not be # responsive. Should be an integer >=2 for privacy, or set to 0 if you diff --git a/joinmarket/support.py b/joinmarket/support.py index 5c991ad7..19daf7a1 100644 --- a/joinmarket/support.py +++ b/joinmarket/support.py @@ -98,6 +98,21 @@ def rand_weighted_choice(n, p_arr): def chunks(d, n): return [d[x:x + n] for x in xrange(0, len(d), n)] +def select_default(unspent, value): + value = int(value) + high = [u for u in unspent if u["value"] >= value] + high.sort(key=lambda u: u["value"]) + low = [u for u in unspent if u["value"] < value] + low.sort(key=lambda u: -u["value"]) + if len(high): + return [high[0]] + i, tv = 0, 0 + while tv < value and i < len(low): + tv += low[i]["value"] + i += 1 + if tv < value: + raise Exception("Not enough funds") + return low[:i] def select_gradual(unspent, value): """ @@ -174,6 +189,21 @@ def select_greediest(unspent, value): end += 1 return low[0:end] +# ordered from most dusty (arguably, most private) to most mergiest (cheaper!) +selectors = [select_default, select_gradual, select_greedy, select_greediest] + +def utxo_selector(configured_levels): + def select(unspent, value): + length = len(unspent) # NB - counted only within each mixdepth + try: + for i in xrange(len(configured_levels)): + if length < configured_levels[i]: + return selectors[i](unspent, value) + return selectors[len(configured_levels)](unspent, value) + except IndexError: + log.debug("Excess merge_algorithm levels. Configure fewer!") + return selectors[-1](unspent, value) # express operator greed + return select def calc_cj_fee(ordertype, cjfee, cj_amount): if ordertype == 'absoffer': diff --git a/joinmarket/wallet.py b/joinmarket/wallet.py index 37720571..59fb1ccc 100644 --- a/joinmarket/wallet.py +++ b/joinmarket/wallet.py @@ -3,6 +3,7 @@ import os import pprint import sys +import ast from decimal import Decimal from ConfigParser import NoSectionError @@ -13,8 +14,7 @@ from joinmarket.blockchaininterface import BitcoinCoreInterface, RegtestBitcoinCoreInterface from joinmarket.configure import jm_single, get_network, get_p2pk_vbyte -from joinmarket.support import get_log, select_gradual, select_greedy, \ - select_greediest +from joinmarket.support import get_log, utxo_selector log = get_log() @@ -46,19 +46,27 @@ def __init__(self): #some consumer scripts don't use an unspent, this marks it #as specifically absent (rather than just empty). self.unspent = None - self.utxo_selector = btc.select # default fallback: upstream try: - config = jm_single().config - if config.get("POLICY", "merge_algorithm") == "gradual": - self.utxo_selector = select_gradual - elif config.get("POLICY", "merge_algorithm") == "greedy": - self.utxo_selector = select_greedy - elif config.get("POLICY", "merge_algorithm") == "greediest": - self.utxo_selector = select_greediest - elif config.get("POLICY", "merge_algorithm") != "default": - raise Exception("Unknown merge algorithm") + policy = jm_single().config.get("POLICY", "merge_algorithm") except NoSectionError: - pass + policy = "default" # maintain backwards compatibility! + if policy == "default": + self.merge_policy = [42] # well, almost (python lacks infinites) + elif policy == "gradual": + self.merge_policy = [60] # never goes beyond gradual + elif policy == "greedy": + self.merge_policy = [70, 70] # skip gradual, go greedy + elif policy == "greediest": + self.merge_policy = [80, 80, 80] # straight to greediest + else: + try: # stop supporting word configs, someday... + self.merge_policy = ast.literal_eval(policy) + if ((type(self.merge_policy) is not list) or + any(type(level) is not int for level in self.merge_policy)): + raise Exception("Merge policy must be a list of ints") + except ValueError: + raise Exception("Unparseable merge policy: "+policy) + self.utxo_selector = utxo_selector(self.merge_policy) def get_key_from_addr(self, addr): return None diff --git a/test/commontest.py b/test/commontest.py index 6060b7d2..5c69b910 100644 --- a/test/commontest.py +++ b/test/commontest.py @@ -147,9 +147,8 @@ def make_wallets(n, 'wallet': w} for j in range(5): for k in range(wallet_structures[i][j]): - deviation = sdev_amt * random.random() - amt = mean_amt - sdev_amt / 2.0 + deviation - if amt < 0: amt = 0.001 + amt = random.gauss(mean_amt, sdev_amt) + if amt < 0.001: amt = 0.001 amt = float(Decimal(amt).quantize(Decimal(10)**-8)) jm_single().bc_interface.grab_coins( wallets[i + start_index]['wallet'].get_external_addr(j), diff --git a/test/regtest_joinmarket.cfg b/test/regtest_joinmarket.cfg index 0f20723e..df3a3b94 100644 --- a/test/regtest_joinmarket.cfg +++ b/test/regtest_joinmarket.cfg @@ -21,11 +21,8 @@ socks5_port = 9150, 9150 [LOGGING] console_log_level = DEBUG [POLICY] -# for dust sweeping, try merge_algorithm = gradual -# for more rapid dust sweeping, try merge_algorithm = greedy -# for most rapid dust sweeping, try merge_algorithm = greediest -# but don't forget to bump your miner fees! -merge_algorithm = default +# same as the default setting +merge_algorithm = [42, 59, 72] # the fee estimate is based on a projection of how many satoshis # per kB are needed to get in one of the next N blocks, N set here # as the value of 'tx_fees'. This estimate can be extremely high diff --git a/test/test_tx_creation.py b/test/test_tx_creation.py index 9f153659..5f94dea1 100644 --- a/test/test_tx_creation.py +++ b/test/test_tx_creation.py @@ -34,7 +34,7 @@ @pytest.mark.parametrize( "nw, wallet_structures, mean_amt, sdev_amt, amount, pubs, k", [ - (1, [[2, 1, 4, 0, 0]], 4, 1.4, 600000000, vpubs[1:4], 2), + (1, [[2, 1, 4, 0, 0]], 6, 1, 600000000, vpubs[1:4], 2), (1, [[3, 3, 0, 0, 3]], 4, 1.4, 100000000, vpubs[:4], 3), ]) def test_create_p2sh_output_tx(setup_tx_creation, nw, wallet_structures, diff --git a/test/test_wallets.py b/test/test_wallets.py index 215b437d..a6822428 100644 --- a/test/test_wallets.py +++ b/test/test_wallets.py @@ -350,6 +350,41 @@ def test_utxo_selection(setup_wallets, nw, wallet_structures, mean_amt, "failed to select sufficient coins, total: " + \ str(total_selected) + ", should be: " + str(amount) +# the following test mainly checks that the algo switching based on +# number of utxos occurs correctly; it also provides some rough sanity +# testing on the general behavior of the merge algorithms. +# TODO: separate & deterministic tests for each merge algo! + +def test_merge_algo_switching(setup_wallets): + wallet = make_wallets(1, [[5, 50, 75, 0, 0]], 1, 0.72)[0]['wallet'] + sync_wallet(wallet), sync_wallet(wallet) + all_utxos = wallet.get_utxos_by_mixdepth() + # test merge-avoiding with small utxo sets - select_default + utxos = sorted(map(lambda x:x['value'], all_utxos[0].values())) + for i in range(4): + amount = (utxos[i]+utxos[i+1])/2 + selected = wallet.select_utxos(0, amount) + assert (1 == len(selected)), "Default selection misbehaved! " + \ + "Selected " + str(selected) + " to reach sum " + str(amount) + \ + " from utxos " + str(utxos) + " (should pick SINGLE utxo)" + # test merging with larger utxo sets - select_gradual + utxos = sorted(map(lambda x:x['value'], all_utxos[1].values())) + for i in range(49): + amount = utxos[i+1]+1 + selected = wallet.select_utxos(1, amount) + assert (1 < len(selected)), "Default selection misbehaved! "+\ + "Selected " + str(selected) + " to reach sum " + str(amount) + \ + " from utxos " + str(utxos) + " (should pick MULTIPLE utxos)" + # TODO: test merging with intermediate sets - select_greedy + # test merging with even larger utxo sets - select_greediest + utxos = sorted(map(lambda x:x['value'], all_utxos[2].values())) + for i in range(74): + amount = sum(utxos[0:i+2]) + selected = wallet.select_utxos(2, amount) + assert (i+2 <= len(selected)), "Default selection misbehaved! "+\ + "Selected " + str(selected) + " to reach sum " + str(amount) + \ + " from utxos " + str(utxos) + " (expected " + str(i+2) + ")" + class TestWalletCreation(unittest.TestCase):