more docs

Pyrlang/Dist/distribution.py

@@ -1,3 +1,7 @@
+""" Distribution class is not a separate running Greenlet, but rather a helper,
+    which is called upon.
+"""
+
 from __future__ import print_function
 import gevent
 from gevent.server import StreamServer
@@ -14,7 +18,11 @@ class ErlangDistribution:
 
     def __init__(self, node, name: str) -> None:
         self.name_ = name
+        """ Node name, a string. """
         self.creation_ = 0
+        """ Creation id used in pid generation. EPMD gives creation id to 
+            newly connected nodes. 
+        """
 
         # Listener for Incoming connections from other nodes
         # Create handler using make_handler helper
@@ -44,9 +52,9 @@ def connect(self, node) -> bool:
 
             gevent.sleep(5)
 
-    def disconnect(self):
-        """
-        :return:
+    def disconnect(self) -> None:
+        """ Finish EPMD connection, this will remove the node from the list of
+            available nodes on EPMD
         """
         self.epmd_.close()
 

Pyrlang/Dist/epmd.py

@@ -1,3 +1,7 @@
+""" The module represents EPMD connection and implements the protocol.
+    EPMD is a daemon application, part of Erlang/OTP which registers Erlang
+    nodes on the local machine and helps nodes finding each other.
+"""
 import struct
 import gevent
 import sys
@@ -15,22 +19,28 @@
 
 
 class ErlEpmd:
-    """ Represents an EPMD client
+    """ An EPMD client connection which registers ourselves in EPMD and can
+        potentially send more commands (TODO).
     """
 
     def __init__(self) -> None:
         self.host_ = '127.0.0.1'
+        """ The local EPMD is always located on the local host. """
         self.port_ = 4369
 
         self.sock_ = None  # network socket
 
     def close(self):
+        """ Closing EPMD connection removes the node from available EPMD nodes
+            list.
+        """
         print("EPMD: Close")
         self.sock_.close()
         self.sock_ = None
 
     def connect(self) -> bool:
         """ A long running connection to EPMD
+
             :return: True
         """
         while True:
@@ -49,7 +59,11 @@ def connect(self) -> bool:
         return True
 
     def alive2(self, dist) -> bool:
-        # Say hello to our EPMD friend
+        """ Send initial hello (ALIVE2) to EPMD
+
+            :param dist: The distribution object from the node
+            :return: Success True or False
+        """
         self._req_alive2(nodetype=NODE_HIDDEN,
                          node_name=dist.name_,
                          in_port=dist.in_port_,
@@ -68,6 +82,7 @@ def alive2(self, dist) -> bool:
 
     def _read_alive2_reply(self) -> int:
         """ Read reply from ALIVE2 request, check the result code, read creation
+
             :return: Creation value if all is well, connection remains on.
                 On error returns -1
         """

Pyrlang/Dist/etf.py

@@ -1,3 +1,6 @@
+""" Module implements encoder and decoder from ETF (Erlang External Term Format)
+    used by the network distribution layer.
+"""
 from __future__ import print_function
 
 import struct
@@ -43,7 +46,7 @@ def incomplete_data(where=""):
 
 
 def binary_to_term(data: bytes):
-    """ Strip 131 header and unpack if the data was compressed
+    """ Strip 131 header and unpack if the data was compressed.
     """
     if data[0] != ETF_VERSION_TAG:
         raise ETFDecodeException("Unsupported external term version")
@@ -63,9 +66,21 @@ def binary_to_term(data: bytes):
 
 
 def binary_to_term_2(data: bytes):
-    """ Proceed decoding after leading tag has been checked and removed
+    """ Proceed decoding after leading tag has been checked and removed.
+
+        Erlang lists are decoded into ``term.List`` object, whose ``elements_``
+        field contains the data, ``tail_`` field has the optional tail and a
+        helper function exists to assist with extracting an unicode string.
+
+        Atoms are decoded into ``term.Atom``. Pids and refs into ``term.Pid``
+        and ``term.Reference`` respectively. Maps are decoded into Python
+        ``dict``. Binaries and bit strings are decoded into ``term.Binary``
+        object, with optional last bits omitted.
+
         :param data: Bytes containing encoded term without 131 header
-        :return: Tuple (Value, TailBytes)
+        :return: Tuple (Value, TailBytes) The function consumes as much data as
+            possible and returns the tail. Tail can be used again to parse
+            another term if there was any.
     """
     tag = data[0]
 
@@ -323,6 +338,13 @@ def _pack_binary(data, last_byte_bits):
 
 
 def term_to_binary_2(val):
+    """ Erlang lists are decoded into term.List object, whose ``elements_``
+        field contains the data, ``tail_`` field has the optional tail and a
+        helper function exists to assist with extracting an unicode string.
+
+        :param val: Almost any Python value
+        :return: bytes object with encoded data, but without a 131 header byte.
+    """
     if type(val) == str:
         return _pack_str(val)
 
@@ -370,7 +392,10 @@ def term_to_binary_2(val):
 
 
 def term_to_binary(val):
+    """ Prepend the 131 header byte to encoded data.
+    """
     return bytes([ETF_VERSION_TAG]) + term_to_binary_2(val)
 
 
-__all__ = ['binary_to_term', 'term_to_binary']
+__all__ = ['binary_to_term', 'binary_to_term_2',
+           'term_to_binary', 'term_to_binary_2']

Pyrlang/Dist/in_connection.py

@@ -1,3 +1,6 @@
+""" The module implements incoming TCP distribution connection (i.e. initiated
+    by another node with the help of EPMD).
+"""
 from __future__ import print_function
 import random
 import struct
@@ -26,8 +29,11 @@ class DistributionError(Exception):
 
 
 class InConnection:
-    """ Handling incoming connections from other nodes.
-        Called and controlled from handler provided by util.make_handler
+    """ Handles incoming connections from other nodes.
+
+        Behaves like a ``Greenlet`` but the actual Greenlet run procedure and
+        the recv loop around this protocol are located in the
+        ``util.make_handler`` helper function.
     """
     DISCONNECTED = 0
     RECV_NAME = 1
@@ -37,29 +43,40 @@ class InConnection:
     def __init__(self, dist, node_opts: NodeOpts):
         self.state_ = self.DISCONNECTED
         self.packet_len_size_ = 2
+        """ Packet size header is variable, 2 bytes before handshake is finished
+            and 4 bytes afterwards.
+        """
         self.socket_ = None
         self.addr_ = None
 
         self.dist_ = dist  # reference to distribution object
         self.node_opts_ = node_opts
         self.inbox_ = Queue()  # refer to util.make_handler which reads this
+        """ Inbox is used to ask the connection to do something. """
 
         self.peer_distr_version_ = (None, None)
+        """ Protocol version range supported by the remote peer. Erlang/OTP 
+            versions 19-20 supports protocol version 7, older Erlangs down to 
+            R6B support version 5. """
         self.peer_flags_ = 0
         self.peer_name_ = None
         self.my_challenge_ = None
 
     def on_connected(self, sockt, address):
+        """ Handler invoked from the recv loop (in ``util.make_handler``)
+            when the connection has been accepted and established.
+        """
         self.state_ = self.RECV_NAME
         self.socket_ = sockt
         self.addr_ = address
 
     def consume(self, data: bytes) -> Union[bytes, None]:
         """ Attempt to consume first part of data as a packet
-        :param data: The accumulated data from the socket which we try to
-            partially or fully consume
-        :return: Unconsumed data, incomplete following packet maybe or nothing
-            Returning None requests to close the connection
+
+            :param data: The accumulated data from the socket which we try to
+                partially or fully consume
+            :return: Unconsumed data, incomplete following packet maybe or
+                nothing. Returning None requests to close the connection
         """
         if len(data) < self.packet_len_size_:
             # Not ready yet, keep reading
@@ -98,7 +115,9 @@ def on_connection_lost(self):
 
     def on_packet(self, data) -> bool:
         """ Handle incoming distribution packet
-        :param data: The packet after the header with length has been removed
+
+            :param data: The packet after the header with length has been
+                removed.
         """
         if self.state_ == self.RECV_NAME:
             return self.on_packet_recvname(data)
@@ -110,7 +129,7 @@ def on_packet(self, data) -> bool:
             return self.on_packet_connected(data)
 
     @staticmethod
-    def error(msg) -> False:
+    def _error(msg) -> False:
         ERROR("Distribution protocol error:", msg)
         return False
 
@@ -122,14 +141,15 @@ def _dist_version_check(pdv: tuple):
         return pdv[0] >= epmd.DIST_VSN >= pdv[1]
 
     def on_packet_recvname(self, data) -> bool:
+        """ Handle RECV_NAME command, the first packet in a new connection. """
         if data[0] != ord('n'):
-            return self.error("Unexpected packet (expecting RECV_NAME)")
+            return self._error("Unexpected packet (expecting RECV_NAME)")
 
         # Read peer distribution version and compare to ours
         pdv = (data[1], data[2])
         if self._dist_version_check(pdv):
-            return self.error("Dist protocol version have: %s got: %s"
-                              % (str(epmd.DIST_VSN_PAIR), str(pdv)))
+            return self._error("Dist protocol version have: %s got: %s"
+                               % (str(epmd.DIST_VSN_PAIR), str(pdv)))
         self.peer_distr_version_ = pdv
 
         self.peer_flags_ = util.u32(data[3:7])
@@ -153,15 +173,15 @@ def on_packet_recvname(self, data) -> bool:
 
     def on_packet_challengereply(self, data):
         if data[0] != ord('r'):
-            return self.error("Unexpected packet (expecting CHALLENGE_REPLY)")
+            return self._error("Unexpected packet (expecting CHALLENGE_REPLY)")
 
         peers_challenge = util.u32(data, 1)
         peer_digest = data[5:]
         LOG("challengereply: peer's challenge", peers_challenge)
 
         my_cookie = self.node_opts_.cookie_
         if not self._check_digest(peer_digest, self.my_challenge_, my_cookie):
-            return self.error("Disallowed node connection (check the cookie)")
+            return self._error("Disallowed node connection (check the cookie)")
 
         self._send_challenge_ack(peers_challenge, my_cookie)
         self.packet_len_size_ = 4
@@ -190,7 +210,7 @@ def on_packet_connected(self, data):
             self.on_passthrough_message(control_term, msg_term)
 
         else:
-            return self.error("Unexpected dist message type: %s" % msg_type)
+            return self._error("Unexpected dist message type: %s" % msg_type)
 
         return True
 

Pyrlang/Dist/node_opts.py

@@ -4,21 +4,28 @@
 DFLAG_PUBLISHED = 0x01
 DFLAG_ATOM_CACHE = 0x02
 DFLAG_EXT_REFS = 0x04
+""" The node supports encoding/decoding of external references. """
 DFLAG_DIST_MONITOR = 0x08
+""" The node supports remote monitoring for processes. """
 DFLAG_FUN_TAGS = 0x10
 DFLAG_DIST_MONITOR_NAME = 0x20
+""" The node supports remote monitoring for named processes. """
 DFLAG_HIDDEN_ATOM_CACHE = 0x40
 DFLAG_NEW_FUN_TAGS = 0x80
 DFLAG_EXT_PIDS_PORTS = 0x100
+""" The node supports encoding/decoding of external pids and ports. """
 DFLAG_EXPORT_PTR_TAG = 0x200
 DFLAG_BIT_BINARIES = 0x400
+""" The node supports incomplete trailing byte in binaries. """
 DFLAG_NEW_FLOATS = 0x800
+""" The node supports 8-byte double encoding as IEEE-double. """
 DFLAG_UNICODE_IO = 0x1000
 DFLAG_DIST_HDR_ATOM_CACHE = 0x2000
 DFLAG_SMALL_ATOM_TAGS = 0x4000
 # 0x8000
 DFLAG_UTF8_ATOMS = 0x10000
 DFLAG_MAP_TAG = 0x20000
+""" The node can handle map encoding. """
 DFLAG_BIG_CREATION = 0x40000
 
 DEFAULT_DFLAGS = (DFLAG_EXT_REFS |
@@ -29,9 +36,14 @@
                   DFLAG_NEW_FLOATS |
                   DFLAG_MAP_TAG |
                   DFLAG_DIST_MONITOR_NAME | DFLAG_DIST_MONITOR)
-
+""" Default flags value represents current Pyrlang library features
+    as a combination of feature bits.
+"""
 
 class NodeOpts:
+    """ A class holding an integer with features that are supported
+        by this node, and the network cookie.
+    """
     def __init__(self, cookie: str, dflags: int = DEFAULT_DFLAGS) -> None:
         self.dflags_ = dflags
         self.cookie_ = cookie

docs/source/Dist.distribution.rst

@@ -0,0 +1,7 @@
+Dist.distribution
+=================
+
+.. automodule:: Pyrlang.Dist.distribution
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/source/Dist.epmd.rst

@@ -0,0 +1,7 @@
+Dist.epmd
+=========
+
+.. automodule:: Pyrlang.Dist.epmd
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/source/Dist.etf.rst

@@ -0,0 +1,7 @@
+Dist.etf
+========
+
+.. automodule:: Pyrlang.Dist.etf
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/source/Dist.in_connection.rst

@@ -0,0 +1,7 @@
+Dist.in_connection
+==================
+
+.. automodule:: Pyrlang.Dist.in_connection
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/source/Dist.node_opts.rst

@@ -0,0 +1,7 @@
+Dist.node_opts
+==============
+
+.. automodule:: Pyrlang.Dist.node_opts
+    :members:
+    :undoc-members:
+    :show-inheritance: