squashed (#1302)

part 2: policy refactor + cli tests

Author: oberstet

.travis.yml

@@ -30,6 +30,11 @@ matrix:
     - python: pypy3
       env: TOX_ENV=pypy3-cli
 
+    - python: 3.6
+      env: TOX_ENV=py36-examples
+    - python: pypy3
+      env: TOX_ENV=pypy3-examples
+
     - python: 3.6
       env: TOX_ENV=py36-unpinned-trial,coverage-report
     - python: 3.6

Makefile

@@ -29,8 +29,9 @@ clean:
 	# Learn to love the shell! http://unix.stackexchange.com/a/115869/52500
 	find . \( -name "*__pycache__" -type d \) -prune -exec rm -rf {} +
 
+# build documentation
 docs:
-	python docs/test_server.py
+	sphinx-build -b html ./docs ./docs/_build
 
 # call this in a fresh virtualenv to update our frozen requirements.txt!
 freeze: clean
@@ -72,16 +73,15 @@ publish: clean
 test: flake8
 	trial crossbar
 
-test_quick:
-	tox -e flake8,py36-abtrunk-trial
+test_tox:
+	tox -e flake8,py36-unpinned-trial,py36-cli .
 
 test_cli:
 	./test/test_cli.sh
 
 test_cli_tox:
 	tox -e py36-cli .
 
-
 test_mqtt:
 #	trial crossbar.adapter.mqtt.test.test_wamp
 	trial crossbar.adapter.mqtt.test.test_wamp.MQTTAdapterTests.test_basic_publish

crossbar/__init__.py

@@ -124,7 +124,7 @@ def run(args=None, reactor=None):
 
     ``--cbdir``         **CROSSBAR_DIR**          Node directory (local directory)
     ``--config``        **CROSSBAR_CONFIG**       Node configuration (local filename)
-    ``--color``        n.a.                      Enable colored terminal output
+    ``--color``         n.a.                      Enable colored terminal output
     ``--loglevel``      n.a.                      Select log level
     ``--logformat``     n.a.                      Select log format
     ``--logdir``        n.a.                      Log to this local directory

crossbar/common/key.py

@@ -41,14 +41,22 @@
 import pkg_resources
 import pyqrcode
 
+from nacl.signing import SigningKey
+from nacl.encoding import HexEncoder
+
+import txaio
+from autobahn.util import utcnow
 from autobahn.wamp.cryptosign import _read_signify_ed25519_pubkey, _qrcode_from_signify_ed25519_pubkey
 
 from twisted.python.runtime import platform
 
 import crossbar
 
 
-def _read_release_pubkey():
+log = txaio.make_logger()
+
+
+def _read_release_key():
     release_pubkey_file = 'crossbar-{}.pub'.format('-'.join(crossbar.__version__.split('.')[0:2]))
     release_pubkey_path = os.path.join(pkg_resources.resource_filename('crossbar', 'keys'), release_pubkey_file)
 
@@ -68,7 +76,7 @@ def _read_release_pubkey():
     return release_pubkey
 
 
-def _parse_keyfile(key_path, private=True):
+def _parse_key_file(key_path, private=True):
     """
     Internal helper. This parses a node.pub or node.priv file and
     returns a dict mapping tags -> values.
@@ -99,41 +107,40 @@ def _parse_keyfile(key_path, private=True):
     return tags
 
 
-def _read_node_pubkey(cbdir, privkey_path=u'key.priv', pubkey_path=u'key.pub'):
-
-    node_pubkey_path = os.path.join(cbdir, pubkey_path)
-
-    if not os.path.exists(node_pubkey_path):
-        raise Exception('no node public key found at {}'.format(node_pubkey_path))
+def _read_node_key(cbdir, privkey_path=u'key.priv', pubkey_path=u'key.pub', private=False):
+    if private:
+        node_key_path = os.path.join(cbdir, privkey_path)
+    else:
+        node_key_path = os.path.join(cbdir, pubkey_path)
 
-    node_pubkey_tags = _parse_keyfile(node_pubkey_path)
+    if not os.path.exists(node_key_path):
+        raise Exception('no node key file found at {}'.format(node_key_path))
 
-    node_pubkey_hex = node_pubkey_tags[u'public-key-ed25519']
+    node_key_tags = _parse_key_file(node_key_path)
 
-    qr = pyqrcode.create(node_pubkey_hex, error='L', mode='binary')
+    if private:
+        node_key_hex = node_key_tags[u'private-key-ed25519']
+    else:
+        node_key_hex = node_key_tags[u'public-key-ed25519']
 
+    qr = pyqrcode.create(node_key_hex, error='L', mode='binary')
     mode = 'text'
-
     if mode == 'text':
-        node_pubkey_qr = qr.terminal()
-
+        node_key_qr = qr.terminal()
     elif mode == 'svg':
         import io
         data_buffer = io.BytesIO()
-
         qr.svg(data_buffer, omithw=True)
-
-        node_pubkey_qr = data_buffer.getvalue()
-
+        node_key_qr = data_buffer.getvalue()
     else:
         raise Exception('logic error')
 
-    node_pubkey = {
-        u'hex': node_pubkey_hex,
-        u'qrcode': node_pubkey_qr
+    node_key = {
+        u'hex': node_key_hex,
+        u'qrcode': node_key_qr
     }
 
-    return node_pubkey
+    return node_key
 
 
 def _machine_id():
@@ -187,3 +194,100 @@ def _write_node_key(filepath, tags, msg):
             if value is None:
                 value = 'unknown'
             f.write(u'{}: {}\n'.format(tag, value))
+
+
+def _maybe_generate_key(cbdir, privfile=u'key.priv', pubfile=u'key.pub'):
+
+    privkey_path = os.path.join(cbdir, privfile)
+    pubkey_path = os.path.join(cbdir, pubfile)
+
+    if os.path.exists(privkey_path):
+
+        # node private key seems to exist already .. check!
+
+        priv_tags = _parse_key_file(privkey_path, private=True)
+        for tag in [u'creator', u'created-at', u'machine-id', u'public-key-ed25519', u'private-key-ed25519']:
+            if tag not in priv_tags:
+                raise Exception("Corrupt node private key file {} - {} tag not found".format(privkey_path, tag))
+
+        privkey_hex = priv_tags[u'private-key-ed25519']
+        privkey = SigningKey(privkey_hex, encoder=HexEncoder)
+        pubkey = privkey.verify_key
+        pubkey_hex = pubkey.encode(encoder=HexEncoder).decode('ascii')
+
+        if priv_tags[u'public-key-ed25519'] != pubkey_hex:
+            raise Exception(
+                ("Inconsistent node private key file {} - public-key-ed25519 doesn't"
+                 " correspond to private-key-ed25519").format(pubkey_path)
+            )
+
+        if os.path.exists(pubkey_path):
+            pub_tags = _parse_key_file(pubkey_path, private=False)
+            for tag in [u'creator', u'created-at', u'machine-id', u'public-key-ed25519']:
+                if tag not in pub_tags:
+                    raise Exception("Corrupt node public key file {} - {} tag not found".format(pubkey_path, tag))
+
+            if pub_tags[u'public-key-ed25519'] != pubkey_hex:
+                raise Exception(
+                    ("Inconsistent node public key file {} - public-key-ed25519 doesn't"
+                     " correspond to private-key-ed25519").format(pubkey_path)
+                )
+        else:
+            log.info(
+                "Node public key file {pub_path} not found - re-creating from node private key file {priv_path}",
+                pub_path=pubkey_path,
+                priv_path=privkey_path,
+            )
+            pub_tags = OrderedDict([
+                (u'creator', priv_tags[u'creator']),
+                (u'created-at', priv_tags[u'created-at']),
+                (u'machine-id', priv_tags[u'machine-id']),
+                (u'public-key-ed25519', pubkey_hex),
+            ])
+            msg = u'Crossbar.io node public key\n\n'
+            _write_node_key(pubkey_path, pub_tags, msg)
+
+        log.info("Node key files exist and are valid")
+        log.debug("Node public key: {hex}", hex=pubkey_hex)
+
+    else:
+        # node private key does not yet exist: generate one
+
+        privkey = SigningKey.generate()
+        privkey_hex = privkey.encode(encoder=HexEncoder).decode('ascii')
+        pubkey = privkey.verify_key
+        pubkey_hex = pubkey.encode(encoder=HexEncoder).decode('ascii')
+
+        # first, write the public file
+        tags = OrderedDict([
+            (u'creator', _creator()),
+            (u'created-at', utcnow()),
+            (u'machine-id', _machine_id()),
+            (u'public-key-ed25519', pubkey_hex),
+        ])
+        msg = u'Crossbar.io node public key\n\n'
+        _write_node_key(pubkey_path, tags, msg)
+
+        # now, add the private key and write the private file
+        tags[u'private-key-ed25519'] = privkey_hex
+        msg = u'Crossbar.io node private key - KEEP THIS SAFE!\n\n'
+        _write_node_key(privkey_path, tags, msg)
+
+        log.info("New node key pair generated!")
+
+    # fix file permissions on node public/private key files
+    # note: we use decimals instead of octals as octal literals have changed between Py2/3
+    #
+    if os.stat(pubkey_path).st_mode & 511 != 420:  # 420 (decimal) == 0644 (octal)
+        os.chmod(pubkey_path, 420)
+        log.info("File permissions on node public key fixed")
+
+    if os.stat(privkey_path).st_mode & 511 != 384:  # 384 (decimal) == 0600 (octal)
+        os.chmod(privkey_path, 384)
+        log.info("File permissions on node private key fixed")
+
+    log.info(
+        'Node key loaded from "{priv_path}"',
+        priv_path=privkey_path,
+    )
+    return privkey