ON-15146: Enable ZF_ATTR shrub_controller_id for enabling tcpdirect stacks to work with a given shrub_controller

Author: krishd-amd

src/include/zf_internal/attr_tmpl.h

@@ -346,6 +346,13 @@ ZF_ATTR(int, phys_address_mode, stable, 0, NULL,
         "\n"
         "0 - Don't enable physical addressing mode. User space sees virtual addresses \n"
         "    which are translated by hardware or in the kernel.")
+
+ZF_ATTR(int, shrub_controller_id, stable, -1, NULL,
+        "zf_vi",
+
+        "Enable shrub controller on this VI.  This is required to enable zf_stacks \n"
+        "to attach to a given shared controller on given x4 platforms. \n")
+
 /**********************************************************************
  * zf_pool attributes.
  */

src/include/zf_internal/private/zf_stack_def.h

@@ -286,6 +286,7 @@ struct zf_stack_impl {
   int sti_udp_ttl;
   int sti_rx_datapath;
   int sti_phys_address_mode;
+  int sti_shrub_controller_id;
   uint64_t sti_log_level;
 
   int n_alts; /* Number of alternatives actually allocated to this VI */

src/lib/zf/private/stack_alloc.c

@@ -319,6 +319,21 @@ static int zf_stack_init_phys_address_mode(struct zf_stack_impl* sti,
   return 0;
 }
 
+static int zf_stack_init_shrub_controller(struct zf_stack_impl* sti,
+                                           struct zf_attr* attr,
+                                           int* shrub_controller_id)
+{
+  zf_stack* st = &sti->st;
+
+  if ( attr->shrub_controller_id < -1 || attr->shrub_controller_id > 9999 ) {
+    zf_log_stack_err(st, "Bad shrub_controller_id; value must be -1 or in range 0 to 9999 \n");
+    return -EINVAL;
+  }
+
+  *shrub_controller_id = attr->shrub_controller_id;
+  return 0;
+}
+
 static int zf_stack_init_ctpio_stack_config(struct zf_stack_impl* sti,
                                             struct zf_attr* attr,
                                             int* ctpio_mode)
@@ -572,14 +587,16 @@ int zf_stack_init_nic_resources(struct zf_stack_impl* sti,
                                 struct zf_attr* attr, int nicno,
                                 int ifindex, zf_if_info* if_cplane_info,
                                 unsigned vi_flags, int ctpio_mode,
-                                int rx_datapath, bool phys_address_mode)
+                                int rx_datapath, bool phys_address_mode,
+                                int shrub_controller_id)
 {
   zf_stack* st = &sti->st;
   struct zf_stack_nic* st_nic = &st->nic[nicno];
   struct zf_stack_res_nic* sti_nic = &sti->nic[nicno];
   ef_pd_flags pd_flags = EF_PD_DEFAULT;
   int rc;
   unsigned long capability_val;
+  char shrub_controller_char[5];
 
   /* Open driver. */
   rc = ef_driver_open(&sti_nic->dh);
@@ -606,6 +623,18 @@ int zf_stack_init_nic_resources(struct zf_stack_impl* sti,
         pd_flags = (ef_pd_flags)(pd_flags | EF_PD_EXPRESS);
   }
 
+  rc = snprintf(shrub_controller_char, sizeof(shrub_controller_id), "%d", shrub_controller_id);
+  if ( rc < 0 || rc >= (int)sizeof(shrub_controller_char) ) {
+    zf_log_stack_err(st, "Failed to format shrub_controller_id\n");
+    return -EINVAL;
+  }
+
+  rc = setenv("EF_SHRUB_CONTROLLER", shrub_controller_char , 1);
+  if ( rc != 0 && shrub_controller_id != -1 ) {
+    zf_log_stack_err(st, "Failed to set EF_SHRUB_CONTROLLER environment variable\n");
+    return -EINVAL;
+  }
+
   if ( phys_address_mode ) {
       rc = ef_vi_capabilities_get_from_pd_flags(
              sti_nic->dh, ifindex, (ef_pd_flags)(pd_flags | EF_PD_PHYS_MODE),
@@ -867,6 +896,11 @@ int zf_stack_alloc(struct zf_attr* attr, struct zf_stack** stack_out)
   rc = zf_stack_init_datapath(sti, attr, &rx_datapath);
   if( rc < 0 )
     goto fail2;
+
+  int shrub_controller_id;
+  rc = zf_stack_init_shrub_controller(sti, attr, &shrub_controller_id);
+  if( rc < 0 )
+    goto fail2;
 
   bool phys_address_mode;
   rc = zf_stack_init_phys_address_mode(sti, attr, &phys_address_mode);
@@ -905,6 +939,7 @@ int zf_stack_alloc(struct zf_attr* attr, struct zf_stack** stack_out)
   sti->sti_log_level = attr->log_level;
   sti->sti_rx_datapath = rx_datapath;
   sti->sti_phys_address_mode = phys_address_mode;
+  sti->sti_shrub_controller_id = shrub_controller_id;
 
   strncpy(sti->sti_ctpio_mode, attr->ctpio_mode, 8);
   if( st->encap_type & EF_CP_ENCAP_F_VLAN )
@@ -958,7 +993,8 @@ int zf_stack_alloc(struct zf_attr* attr, struct zf_stack** stack_out)
 
     rc = zf_stack_init_nic_resources(sti, attr, nicno, hwport_ifindex,
                                      &hwport_cplane_info, vi_flags,
-                                     ctpio_mode, rx_datapath, phys_address_mode);
+                                     ctpio_mode, rx_datapath, phys_address_mode,
+                                     shrub_controller_id);
     if( rc < 0 )
       goto fail3;
   }

src/lib/zf/zf_stackdump.c

@@ -48,6 +48,7 @@ void dump_attributes(SkewPointer<zf_stack_impl> stimpl)
     ? "enterprise"
     : "express"));
   zf_dump("phys_address_mode=%d\n", stimpl->sti_phys_address_mode);
+  zf_dump("shrub_controller_id=%d\n", stimpl->sti_shrub_controller_id);
   zf_dump("pio=%d\n", stimpl->sti_pio);
   zf_dump("reactor_spin_count=%d\n", stimpl->sti_reactor_spin_count);
   zf_dump("tcp_timewait_ms=%d\n", stimpl->sti_tcp_timewait_ms);