Refactoring P2P setup code and set up new DB layer framework (#330)

* Added basic event bus system.

* Better sync guarantees for event firing.

* Properly handle events being fired when there's no handlers for them.

* Basic framework of new storage layer and new p2p framework.

* Added "channel state update" event type to HTLC.

* Implemented most of connection setup and workflow and added some hooks into qln code to support it.

* Removed old P2P connection code replacing it with calls to new system.

Hope it all works this time.

* Create new PeerManager when initing a LitNode.

Also rearranged some things in NewLitNode in qln/init.go to make it clearer what's happening.

* Fixed locking problem in eventbus caused by early return.

* Implemented peer database for new BoltDB storage layer.

* Fixed some issues due to Go having silly semantics regarding pointrs and interfaces.

* Implemented a lot of logging messages, fixed some locking problems.

* Fixed not starting message processor when initiating connections after handshake.

* Removed a couple of unnecssary log messages.

* Tweaked some things in the qln event handlers, trying to get it to work.

* Fixed mismatched peer numbering schemes confusing some message handling code.

* Some tweaks to maybe help with debugging?

* Fixed RemotePeer's QCs getting cleared preemptively, causing lots of problems.

* Removed some more peer-related DB code that's now redundant.

* Added new message handlng infrastructure.

* Patched in most of the network message IO code to the new message processor system.

* Added peer disconnect handler.  Also removed the OmniIn channel that was already unused.

* Fixed some inconsistencies and other bugs in marshalling between new and old messaging systems.

* Replaced OmniOut with (wrapped) calls to the new messaging system.

* Tried to fix "nil elkrem receiver" problem when breaking.

* Made some changes to lnio in preparation for walletmgr.

* Renamed btcutil/btcec to btcutil/koblitz.

* Moved btcutil/koblitz to crypto/koblitz.

* Moved DB code into new hierarchy and renamed lnio to lncore which is a more accurate name.

* Updated btcec package references to koblitz, as missed in previous commit.

* Added back support for UPnP, NAT, and SOCKS5 proxys outbound.

* Make sure we're gotten rid of lit and lit-af on `make clean`.

* Fixed some naming issues.

* Updated references to "log" to new "logging" package.

Also fixed some references to btcec that were there after the rebase for some reason.

* Fixed some issues with the the argument parsing not working right.

* Fix crash(es) involved in Connect RPC call.

* Updated code for saving nicknames.

* Fixed more nickname stuff in DB code.

* Added some serialization/deserialization tests.

Author: delbonis

Makefile

@@ -42,6 +42,7 @@ clean:
 	rm -f cmd/lit-af/lit-af
 	rm -f cmd/glit/glit
 	cd test && ./clean.sh
+	rm -f lit lit-af
 
 test: lit
 	build/env.sh ./gotests.sh

btcutil/address.go

@@ -8,10 +8,9 @@ import (
 	"bytes"
 	"encoding/hex"
 	"errors"
-
 	"github.com/mit-dci/lit/btcutil/base58"
-	"github.com/mit-dci/lit/btcutil/btcec"
 	"github.com/mit-dci/lit/btcutil/chaincfg"
+	"github.com/mit-dci/lit/crypto/koblitz"
 	"github.com/mit-dci/lit/crypto/ripemd160"
 )
 
@@ -292,21 +291,21 @@ const (
 // AddressPubKey is an Address for a pay-to-pubkey transaction.
 type AddressPubKey struct {
 	pubKeyFormat PubKeyFormat
-	pubKey       *btcec.PublicKey
+	pubKey       *koblitz.PublicKey
 	pubKeyHashID byte
 }
 
 // NewAddressPubKey returns a new AddressPubKey which represents a pay-to-pubkey
 // address.  The serializedPubKey parameter must be a valid pubkey and can be
 // uncompressed, compressed, or hybrid.
 func NewAddressPubKey(serializedPubKey []byte, net *chaincfg.Params) (*AddressPubKey, error) {
-	pubKey, err := btcec.ParsePubKey(serializedPubKey, btcec.S256())
+	pubKey, err := koblitz.ParsePubKey(serializedPubKey, koblitz.S256())
 	if err != nil {
 		return nil, err
 	}
 
 	// Set the format of the pubkey.  This probably should be returned
-	// from btcec, but do it here to avoid API churn.  We already know the
+	// from koblitz, but do it here to avoid API churn.  We already know the
 	// pubkey is valid since it parsed above, so it's safe to simply examine
 	// the leading byte to get the format.
 	pkFormat := PKFUncompressed
@@ -397,6 +396,6 @@ func (a *AddressPubKey) AddressPubKeyHash() *AddressPubKeyHash {
 }
 
 // PubKey returns the underlying public key for the address.
-func (a *AddressPubKey) PubKey() *btcec.PublicKey {
+func (a *AddressPubKey) PubKey() *koblitz.PublicKey {
 	return a.pubKey
 }

btcutil/blockchain/compress.go

@@ -5,7 +5,7 @@
 package blockchain
 
 import (
-	"github.com/mit-dci/lit/btcutil/btcec"
+	"github.com/mit-dci/lit/crypto/koblitz"
 	"github.com/mit-dci/lit/btcutil/txscript"
 )
 
@@ -218,7 +218,7 @@ func isPubKey(script []byte) (bool, []byte) {
 
 		// Ensure the public key is valid.
 		serializedPubKey := script[1:34]
-		_, err := btcec.ParsePubKey(serializedPubKey, btcec.S256())
+		_, err := koblitz.ParsePubKey(serializedPubKey, koblitz.S256())
 		if err == nil {
 			return true, serializedPubKey
 		}
@@ -230,7 +230,7 @@ func isPubKey(script []byte) (bool, []byte) {
 
 		// Ensure the public key is valid.
 		serializedPubKey := script[1:66]
-		_, err := btcec.ParsePubKey(serializedPubKey, btcec.S256())
+		_, err := koblitz.ParsePubKey(serializedPubKey, koblitz.S256())
 		if err == nil {
 			return true, serializedPubKey
 		}
@@ -399,7 +399,7 @@ func decompressScript(compressedPkScript []byte, version int32) []byte {
 		compressedKey := make([]byte, 33)
 		compressedKey[0] = byte(encodedScriptSize - 2)
 		copy(compressedKey[1:], compressedPkScript[1:])
-		key, err := btcec.ParsePubKey(compressedKey, btcec.S256())
+		key, err := koblitz.ParsePubKey(compressedKey, koblitz.S256())
 		if err != nil {
 			return nil
 		}

btcutil/hdkeychain/extendedkey.go

@@ -16,14 +16,13 @@ import (
 	"encoding/binary"
 	"errors"
 	"fmt"
-	"math/big"
-
 	"github.com/mit-dci/lit/btcutil"
 	"github.com/mit-dci/lit/btcutil/base58"
-	"github.com/mit-dci/lit/btcutil/btcec"
 	"github.com/mit-dci/lit/btcutil/chaincfg"
 	"github.com/mit-dci/lit/btcutil/chaincfg/chainhash"
 	"github.com/mit-dci/lit/coinparam"
+	"github.com/mit-dci/lit/crypto/koblitz"
+	"math/big"
 )
 
 const (
@@ -145,8 +144,8 @@ func (k *ExtendedKey) pubKeyBytes() []byte {
 	// This is a private extended key, so calculate and memoize the public
 	// key if needed.
 	if len(k.pubKey) == 0 {
-		pkx, pky := btcec.S256().ScalarBaseMult(k.key)
-		pubKey := btcec.PublicKey{Curve: btcec.S256(), X: pkx, Y: pky}
+		pkx, pky := koblitz.S256().ScalarBaseMult(k.key)
+		pubKey := koblitz.PublicKey{Curve: koblitz.S256(), X: pkx, Y: pky}
 		k.pubKey = pubKey.SerializeCompressed()
 	}
 
@@ -249,7 +248,7 @@ func (k *ExtendedKey) Child(i uint32) (*ExtendedKey, error) {
 	// a child extended key can't be created for this index and the caller
 	// should simply increment to the next index.
 	ilNum := new(big.Int).SetBytes(il)
-	if ilNum.Cmp(btcec.S256().N) >= 0 || ilNum.Sign() == 0 {
+	if ilNum.Cmp(koblitz.S256().N) >= 0 || ilNum.Sign() == 0 {
 		return nil, ErrInvalidChild
 	}
 
@@ -271,22 +270,22 @@ func (k *ExtendedKey) Child(i uint32) (*ExtendedKey, error) {
 		// childKey = parse256(Il) + parenKey
 		keyNum := new(big.Int).SetBytes(k.key)
 		ilNum.Add(ilNum, keyNum)
-		ilNum.Mod(ilNum, btcec.S256().N)
+		ilNum.Mod(ilNum, koblitz.S256().N)
 		childKey = ilNum.Bytes()
 		isPrivate = true
 	} else {
 		// Case #3.
 		// Calculate the corresponding intermediate public key for
 		// intermediate private key.
-		ilx, ily := btcec.S256().ScalarBaseMult(il)
+		ilx, ily := koblitz.S256().ScalarBaseMult(il)
 		if ilx.Sign() == 0 || ily.Sign() == 0 {
 			return nil, ErrInvalidChild
 		}
 
 		// Convert the serialized compressed parent public key into X
 		// and Y coordinates so it can be added to the intermediate
 		// public key.
-		pubKey, err := btcec.ParsePubKey(k.key, btcec.S256())
+		pubKey, err := koblitz.ParsePubKey(k.key, koblitz.S256())
 		if err != nil {
 			return nil, err
 		}
@@ -295,8 +294,8 @@ func (k *ExtendedKey) Child(i uint32) (*ExtendedKey, error) {
 		// derive the final child key.
 		//
 		// childKey = serP(point(parse256(Il)) + parentKey)
-		childX, childY := btcec.S256().Add(ilx, ily, pubKey.X, pubKey.Y)
-		pk := btcec.PublicKey{Curve: btcec.S256(), X: childX, Y: childY}
+		childX, childY := koblitz.S256().Add(ilx, ily, pubKey.X, pubKey.Y)
+		pk := koblitz.PublicKey{Curve: koblitz.S256(), X: childX, Y: childY}
 		childKey = pk.SerializeCompressed()
 	}
 
@@ -335,21 +334,21 @@ func (k *ExtendedKey) Neuter() (*ExtendedKey, error) {
 		k.depth, k.childNum, false), nil
 }
 
-// ECPubKey converts the extended key to a btcec public key and returns it.
-func (k *ExtendedKey) ECPubKey() (*btcec.PublicKey, error) {
-	return btcec.ParsePubKey(k.pubKeyBytes(), btcec.S256())
+// ECPubKey converts the extended key to a koblitz public key and returns it.
+func (k *ExtendedKey) ECPubKey() (*koblitz.PublicKey, error) {
+	return koblitz.ParsePubKey(k.pubKeyBytes(), koblitz.S256())
 }
 
-// ECPrivKey converts the extended key to a btcec private key and returns it.
+// ECPrivKey converts the extended key to a koblitz private key and returns it.
 // As you might imagine this is only possible if the extended key is a private
 // extended key (as determined by the IsPrivate function).  The ErrNotPrivExtKey
 // error will be returned if this function is called on a public extended key.
-func (k *ExtendedKey) ECPrivKey() (*btcec.PrivateKey, error) {
+func (k *ExtendedKey) ECPrivKey() (*koblitz.PrivateKey, error) {
 	if !k.isPrivate {
 		return nil, ErrNotPrivExtKey
 	}
 
-	privKey, _ := btcec.PrivKeyFromBytes(btcec.S256(), k.key)
+	privKey, _ := koblitz.PrivKeyFromBytes(koblitz.S256(), k.key)
 	return privKey, nil
 }
 
@@ -471,7 +470,7 @@ func NewMaster(seed []byte, net *coinparam.Params) (*ExtendedKey, error) {
 
 	// Ensure the key in usable.
 	secretKeyNum := new(big.Int).SetBytes(secretKey)
-	if secretKeyNum.Cmp(btcec.S256().N) >= 0 || secretKeyNum.Sign() == 0 {
+	if secretKeyNum.Cmp(koblitz.S256().N) >= 0 || secretKeyNum.Sign() == 0 {
 		return nil, ErrUnusableSeed
 	}
 
@@ -518,13 +517,13 @@ func NewKeyFromString(key string) (*ExtendedKey, error) {
 		// of the order of the secp256k1 curve and not be 0.
 		keyData = keyData[1:]
 		keyNum := new(big.Int).SetBytes(keyData)
-		if keyNum.Cmp(btcec.S256().N) >= 0 || keyNum.Sign() == 0 {
+		if keyNum.Cmp(koblitz.S256().N) >= 0 || keyNum.Sign() == 0 {
 			return nil, ErrUnusableSeed
 		}
 	} else {
 		// Ensure the public key parses correctly and is actually on the
 		// secp256k1 curve.
-		_, err := btcec.ParsePubKey(keyData, btcec.S256())
+		_, err := koblitz.ParsePubKey(keyData, koblitz.S256())
 		if err != nil {
 			return nil, err
 		}

btcutil/txscript/engine.go

@@ -9,7 +9,7 @@ import (
 	"fmt"
 	"math/big"
 
-	"github.com/mit-dci/lit/btcutil/btcec"
+	"github.com/mit-dci/lit/crypto/koblitz"
 	"github.com/mit-dci/lit/crypto/fastsha256"
 	"github.com/mit-dci/lit/wire"
 )
@@ -95,7 +95,7 @@ const (
 )
 
 // halforder is used to tame ECDSA malleability (see BIP0062).
-var halfOrder = new(big.Int).Rsh(btcec.S256().N, 1)
+var halfOrder = new(big.Int).Rsh(koblitz.S256().N, 1)
 
 // Engine is the virtual machine that executes scripts.
 type Engine struct {

btcutil/txscript/opcode.go

@@ -12,7 +12,7 @@ import (
 	"fmt"
 	"hash"
 
-	"github.com/mit-dci/lit/btcutil/btcec"
+	"github.com/mit-dci/lit/crypto/koblitz"
 	"github.com/mit-dci/lit/btcutil/chaincfg/chainhash"
 	"github.com/mit-dci/lit/crypto/fastsha256"
 	"github.com/mit-dci/lit/crypto/ripemd160"
@@ -1932,19 +1932,19 @@ func opcodeCheckSig(op *parsedOpcode, vm *Engine) error {
 		hash = calcSignatureHash(subScript, hashType, &vm.tx, vm.txIdx)
 	}
 
-	pubKey, err := btcec.ParsePubKey(pkBytes, btcec.S256())
+	pubKey, err := koblitz.ParsePubKey(pkBytes, koblitz.S256())
 	if err != nil {
 		vm.dstack.PushBool(false)
 		return nil
 	}
 
-	var signature *btcec.Signature
+	var signature *koblitz.Signature
 	if vm.hasFlag(ScriptVerifyStrictEncoding) ||
 		vm.hasFlag(ScriptVerifyDERSignatures) {
 
-		signature, err = btcec.ParseDERSignature(sigBytes, btcec.S256())
+		signature, err = koblitz.ParseDERSignature(sigBytes, koblitz.S256())
 	} else {
-		signature, err = btcec.ParseSignature(sigBytes, btcec.S256())
+		signature, err = koblitz.ParseSignature(sigBytes, koblitz.S256())
 	}
 	if err != nil {
 		vm.dstack.PushBool(false)
@@ -1987,7 +1987,7 @@ func opcodeCheckSigVerify(op *parsedOpcode, vm *Engine) error {
 // the same signature multiple times when verifying a multisig.
 type parsedSigInfo struct {
 	signature       []byte
-	parsedSignature *btcec.Signature
+	parsedSignature *koblitz.Signature
 	parsed          bool
 }
 
@@ -2122,7 +2122,7 @@ func opcodeCheckMultiSig(op *parsedOpcode, vm *Engine) error {
 		signature := rawSig[:len(rawSig)-1]
 
 		// Only parse and check the signature encoding once.
-		var parsedSig *btcec.Signature
+		var parsedSig *koblitz.Signature
 		if !sigInfo.parsed {
 			if err := vm.checkHashTypeEncoding(hashType); err != nil {
 				return err
@@ -2136,11 +2136,11 @@ func opcodeCheckMultiSig(op *parsedOpcode, vm *Engine) error {
 			if vm.hasFlag(ScriptVerifyStrictEncoding) ||
 				vm.hasFlag(ScriptVerifyDERSignatures) {
 
-				parsedSig, err = btcec.ParseDERSignature(signature,
-					btcec.S256())
+				parsedSig, err = koblitz.ParseDERSignature(signature,
+					koblitz.S256())
 			} else {
-				parsedSig, err = btcec.ParseSignature(signature,
-					btcec.S256())
+				parsedSig, err = koblitz.ParseSignature(signature,
+					koblitz.S256())
 			}
 			sigInfo.parsed = true
 			if err != nil {
@@ -2162,7 +2162,7 @@ func opcodeCheckMultiSig(op *parsedOpcode, vm *Engine) error {
 		}
 
 		// Parse the pubkey.
-		parsedPubKey, err := btcec.ParsePubKey(pubKey, btcec.S256())
+		parsedPubKey, err := koblitz.ParsePubKey(pubKey, koblitz.S256())
 		if err != nil {
 			continue
 		}

btcutil/txscript/sigcache.go

@@ -7,7 +7,7 @@ package txscript
 import (
 	"sync"
 
-	"github.com/mit-dci/lit/btcutil/btcec"
+	"github.com/mit-dci/lit/crypto/koblitz"
 	"github.com/mit-dci/lit/btcutil/chaincfg/chainhash"
 )
 
@@ -18,8 +18,8 @@ import (
 // match. In the occasion that two sigHashes collide, the newer sigHash will
 // simply overwrite the existing entry.
 type sigCacheEntry struct {
-	sig    *btcec.Signature
-	pubKey *btcec.PublicKey
+	sig    *koblitz.Signature
+	pubKey *koblitz.PublicKey
 }
 
 // SigCache implements an ECDSA signature verification cache with a randomized
@@ -55,7 +55,7 @@ func NewSigCache(maxEntries uint) *SigCache {
 //
 // NOTE: This function is safe for concurrent access. Readers won't be blocked
 // unless there exists a writer, adding an entry to the SigCache.
-func (s *SigCache) Exists(sigHash chainhash.Hash, sig *btcec.Signature, pubKey *btcec.PublicKey) bool {
+func (s *SigCache) Exists(sigHash chainhash.Hash, sig *koblitz.Signature, pubKey *koblitz.PublicKey) bool {
 	s.RLock()
 	defer s.RUnlock()
 
@@ -73,7 +73,7 @@ func (s *SigCache) Exists(sigHash chainhash.Hash, sig *btcec.Signature, pubKey *
 //
 // NOTE: This function is safe for concurrent access. Writers will block
 // simultaneous readers until function execution has concluded.
-func (s *SigCache) Add(sigHash chainhash.Hash, sig *btcec.Signature, pubKey *btcec.PublicKey) {
+func (s *SigCache) Add(sigHash chainhash.Hash, sig *koblitz.Signature, pubKey *koblitz.PublicKey) {
 	s.Lock()
 	defer s.Unlock()