Hier neighbor allreduce

bluefog/common/basics.py

@@ -51,7 +51,7 @@ def init(self, topology_fn: Optional[Callable[[int], networkx.DiGraph]] = None,
         Args:
           topology_fn: A callable function that takes size as input and return
             networkx.DiGraph object to decide the topology. If not provided
-            a default power graph (base 2) structure is called.
+            a default exponential graph (base 2) structure is called.
           is_weighted: If set to true, the neighbor ops like (win_update, neighbor_allreduce) will
             execute the weighted average instead, where the weight is the value used in
             topology matrix (including self).
@@ -60,7 +60,7 @@ def init(self, topology_fn: Optional[Callable[[int], networkx.DiGraph]] = None,
         if topology_fn:
             topo = topology_fn(self.size())
         else:
-            topo = topology_util.PowerGraph(self.size())
+            topo = topology_util.ExponentialGraph(self.size())
         self.set_topology(topo, is_weighted)
         atexit.register(self.shutdown)
 
@@ -191,7 +191,7 @@ def set_topology(self, topology: Optional[networkx.DiGraph] = None,
 
         Args:
           Topo: A networkx.DiGraph object to decide the topology. If not provided
-            a default power graph (base 2) structure is used.
+            a default exponential graph (base 2) structure is used.
           is_weighted: If set to true, the win_update and neighbor_allreduce will execute the
             weighted average instead, where the weights are the value used in topology matrix
             (including self weight). Note win_get/win_put/win_accumulate do not use this weight
@@ -207,10 +207,10 @@ def set_topology(self, topology: Optional[networkx.DiGraph] = None,
             >>> bf.set_topology(topology_util.RingGraph(bf.size()))
         """
         if topology is None:
-            topology = topology_util.PowerGraph(size=self.size())
+            topology = topology_util.ExponentialGraph(size=self.size())
             if self.local_rank() == 0:
                 logger.info(
-                    "Topology is not specified. Default Power Two topology is used.")
+                    "Topology is not specified. Default Exponential Two topology is used.")
 
         if not isinstance(topology, networkx.DiGraph):
             raise TypeError("topology must be a networkx.DiGraph obejct.")

bluefog/common/common.h

@@ -278,6 +278,9 @@ struct TensorTableEntry {
   // Boolean value for enabling topology check.
   bool enable_topo_check = false;
 
+  // Boolean value for hierarchical operation or not.
+  bool is_hierarchical = false;
+
   // The ops requires the mutex.
   bool require_mutex = false;
 

bluefog/common/message.cc

@@ -96,6 +96,10 @@ int32_t Request::device() const { return device_; }
 
 void Request::set_device(int32_t value) { device_ = value; }
 
+bool Request::is_hierarchical() const { return is_hierarchical_; }
+
+void Request::set_is_hierarchical(bool value) { is_hierarchical_ = value; }
+
 const std::vector<int64_t>& Request::tensor_shape() const {
   return tensor_shape_;
 }
@@ -118,6 +122,7 @@ void Request_ParseFromWire(Request& request,
   request.set_tensor_name(obj->tensor_name()->str());
   request.set_root_rank(obj->root_rank());
   request.set_device(obj->device());
+  request.set_is_hierarchical(obj->is_hierarchical());
   request.set_tensor_shape(std::vector<int64_t>(obj->tensor_shape()->begin(),
                                                 obj->tensor_shape()->end()));
 }
@@ -137,6 +142,7 @@ void Request_SerializeToWire(const Request& request,
   request_builder.add_tensor_name(tensor_name_wire);
   request_builder.add_root_rank(request.root_rank());
   request_builder.add_device(request.device());
+  request_builder.add_is_hierarchical(request.is_hierarchical());
   request_builder.add_tensor_shape(tensor_shape_wire);
   obj = request_builder.Finish();
 }

bluefog/common/message.h

@@ -70,6 +70,9 @@ class Request {
   int32_t device() const;
   void set_device(int32_t value);
 
+  bool is_hierarchical() const;
+  void set_is_hierarchical(bool value);
+
   const std::vector<int64_t>& tensor_shape() const;
   void set_tensor_shape(const std::vector<int64_t>& value);
   void add_tensor_shape(int64_t value);
@@ -84,6 +87,7 @@ class Request {
   DataType tensor_type_ = DataType::BLUEFOG_UINT8;
   int32_t root_rank_ = 0;
   int32_t device_ = 0;
+  bool is_hierarchical_ = false;
   std::string tensor_name_;
   std::vector<int64_t> tensor_shape_;
 };