From df4c2c126cf587e0c61539f003a5b23ba217585c Mon Sep 17 00:00:00 2001 From: dhayon Date: Thu, 28 Nov 2024 16:03:58 +0200 Subject: [PATCH 1/5] pyverbs: Add extension of mlx5dv_create_flow_matcher Add support for the ib_port attribute, this attribute is needed when creating a matcher over vport to specify its corresponding port. Add ft_type and ib_port enums to support the attribute comp mask. Add support for the RDMA transport domain. Signed-off-by: dhayon Signed-off-by: Edward Srouji --- pyverbs/providers/mlx5/libmlx5.pxd | 1 + pyverbs/providers/mlx5/mlx5_enums.pxd | 6 ++++++ pyverbs/providers/mlx5/mlx5_enums.pyx | 2 ++ pyverbs/providers/mlx5/mlx5dv_flow.pyx | 8 +++++--- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/pyverbs/providers/mlx5/libmlx5.pxd b/pyverbs/providers/mlx5/libmlx5.pxd index 72f8157bd..74eb41e86 100644 --- a/pyverbs/providers/mlx5/libmlx5.pxd +++ b/pyverbs/providers/mlx5/libmlx5.pxd @@ -149,6 +149,7 @@ cdef extern from 'infiniband/mlx5dv.h': mlx5dv_flow_match_parameters *match_mask; uint64_t comp_mask; mlx5_ib_uapi_flow_table_type ft_type; + uint32_t ib_port; cdef struct mlx5dv_flow_matcher diff --git a/pyverbs/providers/mlx5/mlx5_enums.pxd b/pyverbs/providers/mlx5/mlx5_enums.pxd index b9d59dd6e..ec7a7566d 100644 --- a/pyverbs/providers/mlx5/mlx5_enums.pxd +++ b/pyverbs/providers/mlx5/mlx5_enums.pxd @@ -295,6 +295,10 @@ cdef extern from 'infiniband/mlx5dv.h': cpdef enum: MLX5DV_UMEM_MASK_DMABUF + cpdef enum mlx5dv_flow_matcher_attr_mask: + MLX5DV_FLOW_MATCHER_MASK_FT_TYPE + MLX5DV_FLOW_MATCHER_MASK_IB_PORT + cdef unsigned long long MLX5DV_RES_TYPE_QP cdef unsigned long long MLX5DV_RES_TYPE_RWQ cdef unsigned long long MLX5DV_RES_TYPE_DBR @@ -320,6 +324,8 @@ cdef extern from 'infiniband/mlx5_api.h': cdef int MLX5DV_FLOW_TABLE_TYPE_NIC_RX cdef int MLX5DV_FLOW_TABLE_TYPE_NIC_TX cdef int MLX5DV_FLOW_TABLE_TYPE_FDB + cdef int MLX5DV_FLOW_TABLE_TYPE_RDMA_TRANSPORT_RX + cdef int MLX5DV_FLOW_TABLE_TYPE_RDMA_TRANSPORT_TX cdef int MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2 cdef int MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL diff --git a/pyverbs/providers/mlx5/mlx5_enums.pyx b/pyverbs/providers/mlx5/mlx5_enums.pyx index 1d71ba4cd..18861f4a4 100644 --- a/pyverbs/providers/mlx5/mlx5_enums.pyx +++ b/pyverbs/providers/mlx5/mlx5_enums.pyx @@ -23,6 +23,8 @@ MLX5DV_FLOW_TABLE_TYPE_RDMA_TX_ = MLX5DV_FLOW_TABLE_TYPE_RDMA_TX MLX5DV_FLOW_TABLE_TYPE_NIC_RX_ = MLX5DV_FLOW_TABLE_TYPE_NIC_RX MLX5DV_FLOW_TABLE_TYPE_NIC_TX_ = MLX5DV_FLOW_TABLE_TYPE_NIC_TX MLX5DV_FLOW_TABLE_TYPE_FDB_ = MLX5DV_FLOW_TABLE_TYPE_FDB +MLX5DV_FLOW_TABLE_TYPE_RDMA_TRANSPORT_RX_ = MLX5DV_FLOW_TABLE_TYPE_RDMA_TRANSPORT_RX +MLX5DV_FLOW_TABLE_TYPE_RDMA_TRANSPORT_TX_ = MLX5DV_FLOW_TABLE_TYPE_RDMA_TRANSPORT_TX MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2_ = \ MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2 diff --git a/pyverbs/providers/mlx5/mlx5dv_flow.pyx b/pyverbs/providers/mlx5/mlx5dv_flow.pyx index fe2e7524d..607eb575a 100644 --- a/pyverbs/providers/mlx5/mlx5dv_flow.pyx +++ b/pyverbs/providers/mlx5/mlx5dv_flow.pyx @@ -53,7 +53,7 @@ cdef class Mlx5FlowMatchParameters(PyverbsObject): cdef class Mlx5FlowMatcherAttr(PyverbsObject): def __init__(self, Mlx5FlowMatchParameters match_mask, attr_type=v.IBV_FLOW_ATTR_NORMAL, flags=0, priority=0, - match_criteria_enable=0, comp_mask=0, ft_type=0): + match_criteria_enable=0, comp_mask=0, ft_type=0, ib_port=0): """ Initialize a Mlx5FlowMatcherAttr object over an underlying mlx5dv_flow_matcher_attr C object that defines matcher's attributes. @@ -73,8 +73,8 @@ cdef class Mlx5FlowMatcherAttr(PyverbsObject): Bit 3: misc_parameters_2 Bit 4: misc_parameters_3 Bit 5: misc_parameters_4 - :param comp_mask: MLX5DV_FLOW_MATCHER_MASK_FT_TYPE for ft_type (the - only option that is currently supported) + :param comp_mask: MLX5DV_FLOW_MATCHER_MASK_FT_TYPE for ft_type, + MLX5DV_FLOW_MATCHER_MASK_IB_PORT for ib_port. :param ft_type: Specified in which flow table type, the matcher will store the flow rules: MLX5DV_FLOW_TABLE_TYPE_NIC_RX: Specified this matcher will store ingress flow rules. @@ -87,6 +87,7 @@ cdef class Mlx5FlowMatcherAttr(PyverbsObject): rules. MLX5DV_FLOW_TABLE_TYPE_RDMA_TX - matcher will store egress RDMA flow rules. + :param ib_port: Specifies to which vport to attach the matcher. """ super().__init__() self.attr.type = attr_type @@ -96,6 +97,7 @@ cdef class Mlx5FlowMatcherAttr(PyverbsObject): self.attr.match_mask = match_mask.params self.attr.comp_mask = comp_mask self.attr.ft_type = ft_type + self.attr.ib_port = ib_port cdef class Mlx5FlowMatcher(PyverbsObject): From f635400ed880bbea818164e5ad38d93d12f38588 Mon Sep 17 00:00:00 2001 From: Linoy Ganti Date: Wed, 18 Dec 2024 15:09:40 +0200 Subject: [PATCH 2/5] pyverbs: Add FLOW_ACTION_DEST_DEVX to flow action Add pyverb to support MLX5DV_FLOW_ACTION_DEST_DEVX flow action. Signed-off-by: Linoy Ganti Signed-off-by: dhayon Signed-off-by: Edward Srouji --- pyverbs/providers/mlx5/mlx5dv_flow.pyx | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pyverbs/providers/mlx5/mlx5dv_flow.pyx b/pyverbs/providers/mlx5/mlx5dv_flow.pyx index 607eb575a..726948db1 100644 --- a/pyverbs/providers/mlx5/mlx5dv_flow.pyx +++ b/pyverbs/providers/mlx5/mlx5dv_flow.pyx @@ -6,7 +6,7 @@ from libc.string cimport memcpy from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError, \ PyverbsUserError -from pyverbs.device cimport Context +from pyverbs.providers.mlx5.mlx5dv cimport Mlx5DevxObj from pyverbs.base import PyverbsRDMAErrno from pyverbs.base cimport close_weakrefs from pyverbs.device cimport Context @@ -168,7 +168,7 @@ cdef class Mlx5PacketReformatFlowAction(FlowAction): cdef class Mlx5FlowActionAttr(PyverbsObject): def __init__(self, action_type=None, QP qp=None, - FlowAction flow_action=None): + FlowAction flow_action=None, Mlx5DevxObj obj=None): """ Initialize a Mlx5FlowActionAttr object over an underlying mlx5dv_flow_action_attr C object that defines actions attributes for @@ -176,6 +176,7 @@ cdef class Mlx5FlowActionAttr(PyverbsObject): :param action_type: Type of the action :param qp: A QP target for go to QP action :param flow_action: An action to perform for the flow + :param obj: DEVX object """ super().__init__() if action_type: @@ -186,6 +187,8 @@ cdef class Mlx5FlowActionAttr(PyverbsObject): elif action_type == dv.MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION: self.attr.action = flow_action.action self.action = flow_action + elif action_type == dv.MLX5DV_FLOW_ACTION_DEST_DEVX: + self.attr.obj = obj.obj elif action_type: raise PyverbsUserError(f'Unsupported action type: {action_type}.') @@ -252,7 +255,8 @@ cdef class Mlx5Flow(Flow): if (attr).attr.type == dv.MLX5DV_FLOW_ACTION_DEST_IBV_QP: ((attr.qp)).add_ref(self) self.qp = (attr).qp - elif (attr).attr.type not in [dv.MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION]: + elif (attr).attr.type not in \ + [dv.MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION, dv.MLX5DV_FLOW_ACTION_DEST_DEVX]: raise PyverbsUserError(f'Unsupported action type: ' f'{attr).attr.type}.') memcpy(tmp_addr, &(attr).attr, From b2af890071fe684c9551c660e45b95df95990e24 Mon Sep 17 00:00:00 2001 From: dhayon Date: Sun, 22 Dec 2024 17:01:33 +0200 Subject: [PATCH 3/5] tests: Rename FlowTableEntryMatchParam Renamed the class from FlowTableEntryMatchParam to FlowTableEntryMatchParamSW, to better reflect the differentiation in parameter structures used in SW steering. Signed-off-by: dhayon Signed-off-by: Edward Srouji --- tests/mlx5_prm_structs.py | 2 +- tests/test_mlx5_dr.py | 72 +++++++++++++++++++-------------------- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/tests/mlx5_prm_structs.py b/tests/mlx5_prm_structs.py index a547ea609..371eb4f08 100644 --- a/tests/mlx5_prm_structs.py +++ b/tests/mlx5_prm_structs.py @@ -1501,7 +1501,7 @@ class FlowTableEntryMatchSetMisc5(PRMPacket): ] -class FlowTableEntryMatchParam(PRMPacket): +class FlowTableEntryMatchParamSW(PRMPacket): fields_desc = [ PacketField('outer_headers', FlowTableEntryMatchSetLyr24(), FlowTableEntryMatchSetLyr24), PacketField('misc_parameters', FlowTableEntryMatchSetMisc(), FlowTableEntryMatchSetMisc), diff --git a/tests/test_mlx5_dr.py b/tests/test_mlx5_dr.py index 2ccdd80cf..4fce4ce8e 100644 --- a/tests/test_mlx5_dr.py +++ b/tests/test_mlx5_dr.py @@ -202,8 +202,8 @@ def create_rx_recv_rules_based_on_match_params(self, mask_param, val_param, acti Creates a rule on RX domain that forwards packets that match on the provided parameters to the SW steering flow table and another rule on that table with provided actions. - :param mask_param: The FlowTableEntryMatchParam mask matcher value. - :param val_param: The FlowTableEntryMatchParam value matcher value. + :param mask_param: The FlowTableEntryMatchParamSW mask matcher value. + :param val_param: The FlowTableEntryMatchParamSW value matcher value. :param actions: List of actions to attach to the recv rule. :param match_criteria: the match criteria enable flag to match on :param domain: RX DR domain to use if provided, otherwise create default RX domain. @@ -311,11 +311,11 @@ def dest_port(self, is_vport=True): @staticmethod def create_dest_mac_params(): - from tests.mlx5_prm_structs import FlowTableEntryMatchParam + from tests.mlx5_prm_structs import FlowTableEntryMatchParamSW - eth_match_mask = FlowTableEntryMatchParam() + eth_match_mask = FlowTableEntryMatchParamSW() eth_match_mask.outer_headers.dmac = PacketConsts.MAC_MASK - eth_match_value = FlowTableEntryMatchParam() + eth_match_value = FlowTableEntryMatchParamSW() eth_match_value.outer_headers.dmac = PacketConsts.DST_MAC mask_param = Mlx5FlowMatchParameters(len(eth_match_mask), eth_match_mask) value_param = Mlx5FlowMatchParameters(len(eth_match_value), eth_match_value) @@ -368,11 +368,11 @@ def gen_geneve_tunnel_encap_header(msg_size, is_l2_tunnel=True): @staticmethod def create_geneve_params(): - from tests.mlx5_prm_structs import FlowTableEntryMatchParam - geneve_mask = FlowTableEntryMatchParam() + from tests.mlx5_prm_structs import FlowTableEntryMatchParamSW + geneve_mask = FlowTableEntryMatchParamSW() geneve_mask.misc_parameters.geneve_vni = 0xffffff geneve_mask.misc_parameters.geneve_oam = 1 - geneve_value = FlowTableEntryMatchParam() + geneve_value = FlowTableEntryMatchParamSW() geneve_value.misc_parameters.geneve_vni = PacketConsts.GENEVE_VNI geneve_value.misc_parameters.geneve_oam = PacketConsts.GENEVE_OAM mask_param = Mlx5FlowMatchParameters(len(geneve_mask), geneve_mask) @@ -391,12 +391,12 @@ def gen_roce_bth_header(msg_size): @staticmethod def create_roce_bth_params(): - from tests.mlx5_prm_structs import FlowTableEntryMatchParam - roce_mask = FlowTableEntryMatchParam() + from tests.mlx5_prm_structs import FlowTableEntryMatchParamSW + roce_mask = FlowTableEntryMatchParamSW() roce_mask.misc_parameters.bth_opcode = 0xff roce_mask.misc_parameters.bth_dst_qp = 0xffffff roce_mask.misc_parameters.bth_a = 0x1 - roce_value = FlowTableEntryMatchParam() + roce_value = FlowTableEntryMatchParamSW() roce_value.misc_parameters.bth_opcode = PacketConsts.BTH_OPCODE roce_value.misc_parameters.bth_dst_qp = PacketConsts.BTH_DST_QP roce_value.misc_parameters.bth_a = PacketConsts.BTH_A @@ -409,10 +409,10 @@ def create_empty_matcher_go_to_tbl(self, src_tbl, dst_tbl): Create rule that forward all packets (by empty matcher) from src_tbl to dst_tbl. """ - from tests.mlx5_prm_structs import FlowTableEntryMatchParam + from tests.mlx5_prm_structs import FlowTableEntryMatchParamSW - empty_param = Mlx5FlowMatchParameters(len(FlowTableEntryMatchParam()), - FlowTableEntryMatchParam()) + empty_param = Mlx5FlowMatchParameters(len(FlowTableEntryMatchParamSW()), + FlowTableEntryMatchParamSW()) matcher = DrMatcher(src_tbl, 0, u.MatchCriteriaEnable.NONE, empty_param) go_to_tbl_action = DrActionDestTable(dst_tbl) self.rules.append(DrRule(matcher, empty_param, [go_to_tbl_action])) @@ -530,16 +530,16 @@ def test_metadata_modify_action_set_copy_match(self): Match reg_c_0 and reg_c_1: Rule: prio 0 - val REG_C_DATA. Action: Go To QP """ - from tests.mlx5_prm_structs import FlowTableEntryMatchParam, FlowTableEntryMatchSetMisc2, \ + from tests.mlx5_prm_structs import FlowTableEntryMatchParamSW, FlowTableEntryMatchSetMisc2, \ SetActionIn, CopyActionIn self.create_players(Mlx5DrResources) - match_param = FlowTableEntryMatchParam() + match_param = FlowTableEntryMatchParamSW() empty_param = Mlx5FlowMatchParameters(len(match_param), match_param) - mask_metadata = FlowTableEntryMatchParam(misc_parameters_2= + mask_metadata = FlowTableEntryMatchParamSW(misc_parameters_2= FlowTableEntryMatchSetMisc2(metadata_reg_c_0=0xffff, metadata_reg_c_1=0xffff)) mask_param = Mlx5FlowMatchParameters(len(match_param), mask_metadata) - value_metadata = FlowTableEntryMatchParam(misc_parameters_2= + value_metadata = FlowTableEntryMatchParamSW(misc_parameters_2= FlowTableEntryMatchSetMisc2(metadata_reg_c_0=REG_C_DATA, metadata_reg_c_1=REG_C_DATA)) value_param = Mlx5FlowMatchParameters(len(match_param), value_metadata) @@ -620,13 +620,13 @@ def test_prevent_duplicate_rule(self): Creates RX domain, sets duplicate rule to be not allowed on that domain, try creating duplicate rule. Fail if creation succeeded. """ - from tests.mlx5_prm_structs import FlowTableEntryMatchParam + from tests.mlx5_prm_structs import FlowTableEntryMatchParamSW self.server = Mlx5DrResources(**self.dev_info) domain_rx = DrDomain(self.server.ctx, dve.MLX5DV_DR_DOMAIN_TYPE_NIC_RX) domain_rx.allow_duplicate_rules(False) table = DrTable(domain_rx, 1) - empty_param = Mlx5FlowMatchParameters(len(FlowTableEntryMatchParam()), - FlowTableEntryMatchParam()) + empty_param = Mlx5FlowMatchParameters(len(FlowTableEntryMatchParamSW()), + FlowTableEntryMatchParamSW()) matcher = DrMatcher(table, 0, u.MatchCriteriaEnable.NONE, empty_param) self.qp_action = DrActionQp(self.server.qp) self.drop_action = DrActionDrop() @@ -1051,7 +1051,7 @@ def test_geneve_match_tx(self): Creates matcher on TX to match on Geneve related fields with counter action, sends packets and verifies the matcher. """ - from tests.mlx5_prm_structs import FlowTableEntryMatchParam + from tests.mlx5_prm_structs import FlowTableEntryMatchParamSW self.create_players(Mlx5DrResources) skip_if_has_geneve_tx_bug(self.client.ctx) geneve_mask, geneve_val = self.create_geneve_params() @@ -1069,8 +1069,8 @@ def test_geneve_match_tx(self): # RX domain_rx = DrDomain(self.server.ctx, dve.MLX5DV_DR_DOMAIN_TYPE_NIC_RX) self.qp_action = DrActionQp(self.server.qp) - empty_param = Mlx5FlowMatchParameters(len(FlowTableEntryMatchParam()), - FlowTableEntryMatchParam()) + empty_param = Mlx5FlowMatchParameters(len(FlowTableEntryMatchParamSW()), + FlowTableEntryMatchParamSW()) self.create_rx_recv_rules_based_on_match_params\ (empty_param, empty_param, [self.qp_action], match_criteria=u.MatchCriteriaEnable.NONE, domain=domain_rx) @@ -1095,11 +1095,11 @@ def roce_bth_match(self, domain_flag=dve.MLX5DV_DR_DOMAIN_TYPE_NIC_RX): to the matcher and validate the result. :param domain_flag: RX/TX Domain for the test. """ - from tests.mlx5_prm_structs import FlowTableEntryMatchParam + from tests.mlx5_prm_structs import FlowTableEntryMatchParamSW self.create_players(Mlx5DrResources) roce_bth_mask, roce_bth_val = self.create_roce_bth_params() - empty_param = Mlx5FlowMatchParameters(len(FlowTableEntryMatchParam()), - FlowTableEntryMatchParam()) + empty_param = Mlx5FlowMatchParameters(len(FlowTableEntryMatchParamSW()), + FlowTableEntryMatchParamSW()) self.domain = DrDomain(self.server.ctx, domain_flag) root_table = DrTable(self.domain, 0) root_matcher = DrMatcher(root_table, 0, u.MatchCriteriaEnable.NONE, empty_param) @@ -1247,21 +1247,21 @@ def test_flow_meter(self): red counters to the meter rules to verify the packets split to different colors. Send minimal traffic to see that both counters increased. """ - from tests.mlx5_prm_structs import FlowTableEntryMatchParam, FlowTableEntryMatchSetMisc2,\ + from tests.mlx5_prm_structs import FlowTableEntryMatchParamSW, FlowTableEntryMatchSetMisc2,\ FlowMeterParams self.create_players(Mlx5DrResources) # Common resources - matcher_len = len(FlowTableEntryMatchParam()) - empty_param = Mlx5FlowMatchParameters(matcher_len, FlowTableEntryMatchParam()) + matcher_len = len(FlowTableEntryMatchParamSW()) + empty_param = Mlx5FlowMatchParameters(matcher_len, FlowTableEntryMatchParamSW()) reg_c_idx = self.client.get_first_flow_meter_reg_id() reg_c_field = METADATA_C_FIELDS[reg_c_idx] meter_param = FlowMeterParams(valid=0x1, bucket_overflow=0x1, start_color=0x2, cir_mantissa=1, cir_exponent=6) # 15.625MBps - reg_c_mask = Mlx5FlowMatchParameters(matcher_len, FlowTableEntryMatchParam( + reg_c_mask = Mlx5FlowMatchParameters(matcher_len, FlowTableEntryMatchParamSW( misc_parameters_2=FlowTableEntryMatchSetMisc2(**{reg_c_field: 0xffffffff}))) - reg_c_green = Mlx5FlowMatchParameters(matcher_len, FlowTableEntryMatchParam( + reg_c_green = Mlx5FlowMatchParameters(matcher_len, FlowTableEntryMatchParamSW( misc_parameters_2=FlowTableEntryMatchSetMisc2(**{reg_c_field: FLOW_METER_GREEN}))) - reg_c_red = Mlx5FlowMatchParameters(matcher_len, FlowTableEntryMatchParam( + reg_c_red = Mlx5FlowMatchParameters(matcher_len, FlowTableEntryMatchParamSW( misc_parameters_2=FlowTableEntryMatchSetMisc2(**{reg_c_field: FLOW_METER_RED}))) self.client.domain = DrDomain(self.client.ctx, dve.MLX5DV_DR_DOMAIN_TYPE_NIC_TX) @@ -1308,9 +1308,9 @@ def fwd_packets_to_table(self, src_table, dst_table): :param dst_table: Destination table :return: DrActionDestTable used to move the packets from src_table to dst_table """ - from tests.mlx5_prm_structs import FlowTableEntryMatchParam - empty_param = Mlx5FlowMatchParameters(len(FlowTableEntryMatchParam()), - FlowTableEntryMatchParam()) + from tests.mlx5_prm_structs import FlowTableEntryMatchParamSW + empty_param = Mlx5FlowMatchParameters(len(FlowTableEntryMatchParamSW()), + FlowTableEntryMatchParamSW()) matcher = DrMatcher(src_table, 0, u.MatchCriteriaEnable.NONE, empty_param) dest_table_action = DrActionDestTable(dst_table) self.rules.append(DrRule(matcher, empty_param, [dest_table_action])) From fb811423bb9d4ad35e099f563d2a0a6b794a12dd Mon Sep 17 00:00:00 2001 From: dhayon Date: Sun, 22 Dec 2024 17:42:05 +0200 Subject: [PATCH 4/5] tests: Add test to cover RDMA transport domain Add a test that verifies the creation of the RDMA transport domain and ensures the rule on this table is correctly applied. Updated `tests/mlx5_prm_structs.py` to include the necessary PRM structure definitions for the RDMA transport domain. Signed-off-by: dhayo Signed-off-by: Edward Srouji --- tests/mlx5_base.py | 53 +++--- tests/mlx5_prm_structs.py | 349 +++++++++++++++++++++++++++++++++++++- tests/test_mlx5_flow.py | 223 +++++++++++++++++++++++- 3 files changed, 596 insertions(+), 29 deletions(-) diff --git a/tests/mlx5_base.py b/tests/mlx5_base.py index 2ad79113e..db7ccb1b1 100644 --- a/tests/mlx5_base.py +++ b/tests/mlx5_base.py @@ -1047,6 +1047,35 @@ def create_cq(self): raise ex +def create_privileged_context(agr_obj): + """ + Creates a mlx5 privileged context. + :param agr_obj: Aggregation object which contains all resources necessary. + """ + fds_paths = ['/dev/infiniband/mlx5_perm_ctrl_local', + '/dev/infiniband/mlx5_perm_ctrl_other_vhca'] + open_fds = [] + try: + for file_path in fds_paths: + fd = open(file_path, 'rb') + open_fds.append(fd) + agr_obj.fds = FdArr(arr=[fd.fileno() for fd in open_fds], count=len(open_fds)) + attr = Mlx5DVContextAttr(flags=dve.MLX5DV_CONTEXT_FLAGS_DEVX, + comp_mask=dve.MLX5DV_CONTEXT_ATTR_MASK_FD_ARRAY, + fds=agr_obj.fds) + agr_obj.ctx = Mlx5Context(attr, agr_obj.dev_name) + for fd in open_fds: fd.close() + except FileNotFoundError: + for fd in open_fds: fd.close() + raise unittest.SkipTest(f'FDs file not found') + except PyverbsUserError as ex: + for fd in open_fds: fd.close() + raise unittest.SkipTest(f'Could not open mlx5 context ({ex})') + except PyverbsRDMAError as ex: + for fd in open_fds: fd.close() + raise unittest.SkipTest(f'Opening mlx5 DevX context is not supported ({ex})') + + class Mlx5PrivilegedRC(Mlx5RcResources): def __init__(self, dev_name, ib_port, gid_index, msg_size=1024, **kwargs): """ @@ -1061,29 +1090,7 @@ def __init__(self, dev_name, ib_port, gid_index, msg_size=1024, **kwargs): super().__init__(dev_name, ib_port, gid_index, msg_size=msg_size, **kwargs) def create_context(self): - fds_paths = ['/dev/infiniband/mlx5_perm_ctrl_local', - '/dev/infiniband/mlx5_perm_ctrl_other_vhca'] - fd_array = [] - try: - for file_path in fds_paths: - fd = open(file_path, 'rb') - self.open_fds.append(fd) - fd_array.append(fd.fileno()) - self.fds = FdArr(arr=fd_array, count=len(fd_array)) - attr = Mlx5DVContextAttr(flags=dve.MLX5DV_CONTEXT_FLAGS_DEVX, - comp_mask=dve.MLX5DV_CONTEXT_ATTR_MASK_FD_ARRAY, - fds=self.fds) - self.ctx = Mlx5Context(attr, self.dev_name) - self.close_fds() - except FileNotFoundError: - self.close_fds() - raise unittest.SkipTest(f'FDs file not found') - except PyverbsUserError as ex: - self.close_fds() - raise unittest.SkipTest(f'Could not open mlx5 context ({ex})') - except PyverbsRDMAError as ex: - self.close_fds() - raise unittest.SkipTest(f'Opening mlx5 DevX context is not supported ({ex})') + return create_privileged_context(self) def close_fds(self): for open_fd in self.open_fds: diff --git a/tests/mlx5_prm_structs.py b/tests/mlx5_prm_structs.py index 371eb4f08..16eaada16 100644 --- a/tests/mlx5_prm_structs.py +++ b/tests/mlx5_prm_structs.py @@ -41,6 +41,9 @@ class DevxOps: MLX5_CMD_OP_DEALLOC_FLOW_COUNTER = 0x93a MLX5_CMD_OP_QUERY_FLOW_COUNTER = 0x93b MLX5_CMD_OP_CREATE_TIR = 0x900 + MLX5_CMD_OP_CREATE_FLOW_TABLE = 0x930 + MLX5_CMD_OP_CREATE_FLOW_GROUP = 0x933 + MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY = 0x936 MLX5_CMD_OP_CREATE_EQ = 0x301 MLX5_CMD_OP_MAD_IFC = 0x50d MLX5_CMD_OP_ACCESS_REGISTER_PAOS = 0x5006 @@ -776,6 +779,7 @@ class QueryHcaVportGidOut(PRMPacket): class QueryHcaCapOp: HCA_CAP_2 = 0X20 + ADV_RDMA_CAP = 0x28 HCA_NIC_FLOW_TABLE_CAP = 0x7 @@ -1211,7 +1215,8 @@ class CmdHcaCap(PRMPacket): BitField('reserved51', 0, 3), BitField('max_geneve_tlv_option_data_len', 0, 5), BitField('flex_parser_header_modify', 0, 1), - BitField('reserved52', 0, 2), + BitField('adv_rdma_cap', 0, 1), + BitField('reserved52', 0, 1), BitField('log_max_guaranteed_connections', 0, 5), BitField('reserved53', 0, 3), BitField('log_max_dct_connections', 0, 5), @@ -1501,6 +1506,26 @@ class FlowTableEntryMatchSetMisc5(PRMPacket): ] +class FlowTableEntryMatchSetMisc6(PRMPacket): + fields_desc = [ + FieldListField('nisp_header', [0 for x in range(10)], IntField('', 0), count_from=lambda pkt:10), + StrFixedLenField('reserved1', None, length=24), + ] + + +class FlowTableEntryMatchParam(PRMPacket): + fields_desc = [ + PacketField('outer_headers', FlowTableEntryMatchSetLyr24(), FlowTableEntryMatchSetLyr24), + PacketField('misc_parameters', FlowTableEntryMatchSetMisc(), FlowTableEntryMatchSetMisc), + PacketField('inner_headers', FlowTableEntryMatchSetLyr24(), FlowTableEntryMatchSetLyr24), + PacketField('misc_parameters_2', FlowTableEntryMatchSetMisc2(), FlowTableEntryMatchSetMisc2), + PacketField('misc_parameters_3', FlowTableEntryMatchSetMisc3(), FlowTableEntryMatchSetMisc3), + PacketField('misc_parameters_4', FlowTableEntryMatchSetMisc4(), FlowTableEntryMatchSetMisc4), + PacketField('misc_parameters_5', FlowTableEntryMatchSetMisc5(), FlowTableEntryMatchSetMisc5), + PacketField('misc_parameters_6', FlowTableEntryMatchSetMisc6(), FlowTableEntryMatchSetMisc6), + ] + + class FlowTableEntryMatchParamSW(PRMPacket): fields_desc = [ PacketField('outer_headers', FlowTableEntryMatchSetLyr24(), FlowTableEntryMatchSetLyr24), @@ -2297,6 +2322,44 @@ class FlowTablePropLayout(PRMPacket): FlowTableFieldsSupported), ] +class AdvRdmaCapabilities(PRMPacket): + fields_desc = [ + BitField('ps_cap', 0, 1), + BitField('ps_auto_select', 0, 1), + BitField('reserved1', 0, 30), + ByteField('rcx_type', 0), + BitField('reserved2', 0, 2), + BitField('ps_entry_log_max_value', 0, 6), + BitField('reserved3', 0, 6), + BitField('qp_max_ps_num_entry', 0, 10), + ByteField('mp_max_num_queues', 0), + ByteField('ps_user_context_max_log_size', 0), + ByteField('message_based_qp_and_striding_wq', 0), + ByteField('reserved4', 0), + ShortField('max_receive_send_message_size_stride', 0), + ShortField('reserved5', 0), + IntField('max_receive_send_message_size_byte', 0), + StrFixedLenField('reserved6', None, length=44), + PacketField('rdma_transport_rx_flow_table_properties', FlowTablePropLayout(), FlowTablePropLayout), + PacketField('rdma_transport_tx_flow_table_properties', FlowTablePropLayout(), FlowTablePropLayout), + PacketField('rdma_transport_rx_ft_field_support_2', FlowTableFieldsSupported2(), FlowTableFieldsSupported2), + PacketField('rdma_transport_tx_ft_field_support_2', FlowTableFieldsSupported2(), FlowTableFieldsSupported2), + PacketField('rdma_transport_rx_ft_field_bitmask_support_2', FlowTableFieldsSupported2(), FlowTableFieldsSupported2), + PacketField('rdma_transport_tx_ft_field_bitmask_support_2', FlowTableFieldsSupported2(), FlowTableFieldsSupported2), + PacketField('rdma_transport_rx_header_modify', HeaderModifyCapProperties(), HeaderModifyCapProperties), + PacketField('rdma_transport_tx_header_modify', HeaderModifyCapProperties(), HeaderModifyCapProperties), + ] + + +class QueryAdvRdmaCapOut(PRMPacket): + fields_desc = [ + ByteField('status', 0), + BitField('reserved1', 0, 24), + IntField('syndrome', 0), + StrFixedLenField('reserved2', None, length=8), + PadField(PacketField('capability', AdvRdmaCapabilities(), AdvRdmaCapabilities), 2048, padwith=b"\x00"), + ] + class FlowTableNicCap(PRMPacket): fields_desc = [ @@ -2429,3 +2492,287 @@ class CreateGeneralObjOut(PRMPacket): fields_desc = [ PacketField('general_obj_out_cmd_hdr', GeneralObjOutCmdHdr(), GeneralObjOutCmdHdr), ] + + +class SwOwnerIcmRootParams(PRMPacket): + fields_desc = [ + LongField('sw_owner_icm_root_1', 0), + LongField('sw_owner_icm_root_0', 0), + ] + + +class RtcParams(PRMPacket): + fields_desc = [ + IntField('rtc_id_0', 0), + IntField('rtc_id_1', 0), + StrFixedLenField('reserved1', None, length=8), + ] + + +class FlowTableContext(PRMPacket): + fields_desc = [ + BitField("reformat_en", 0, 1), + BitField("decap_en", 0, 1), + BitField("sw_owner", 0, 1), + BitField("termination_table", 0, 1), + BitField("table_miss_action", 0, 4), + ByteField("level", 0), + BitField("rtc_vld", 0, 1), + BitField("freeze", 0, 1), + BitField("reserved1", 0, 6), + ByteField("log_size", 0), + ByteField("reserved2", 0), + BitField("table_miss_id", 0, 24), + ByteField("reserved3", 0), + BitField("lag_master_next_table_id", 0, 24), + StrFixedLenField('reserved4', None, length=12), + ConditionalField( + PadField(PacketField('sw_owner_icm_root', SwOwnerIcmRootParams(), SwOwnerIcmRootParams), 16, padwith=b"\x00"), + lambda pkt: pkt.sw_owner == 1 or pkt.rtc_vld == 0), + ConditionalField( + PadField(PacketField('rtc_params', RtcParams(), RtcParams), 16, padwith=b"\x00"), + lambda pkt: pkt.rtc_vld == 1), + ] + + +class CreateFlowTableIn(PRMPacket): + fields_desc = [ + ShortField('opcode', DevxOps.MLX5_CMD_OP_CREATE_FLOW_TABLE), + ShortField('uid', 0), + ShortField('vhca_tunnel_id', 0), + ShortField('op_mod', 0), + BitField('other_vport', 0, 1), + BitField('reserved1', 0, 15), + ShortField('vport_number', 0), + StrFixedLenField('reserved2', None, length=4), + ByteField('table_type', 0), + BitField('reserved3', 0, 24), + StrFixedLenField('reserved4', None, length=4), + PacketField('flow_table_context', FlowTableContext(), FlowTableContext), + ] + + +class CreateFlowTableOut(PRMPacket): + fields_desc = [ + ByteField('status', 0), + BitField('icm_address_63_40', 0, 24), + IntField('syndrome', 0), + ByteField('icm_address_39_32', 0), + BitField('table_id', 0, 24), + IntField('icm_address_31_0', 0), + ] + + +class CreateFlowGroupIn(PRMPacket): + fields_desc = [ + ShortField('opcode', DevxOps.MLX5_CMD_OP_CREATE_FLOW_GROUP), + ShortField('uid', 0), + ShortField('vhca_tunnel_id', 0), + ShortField('op_mod', 0), + BitField('other_vport', 0, 1), + BitField('reserved1', 0, 15), + ShortField('vport_number', 0), + StrFixedLenField('reserved2', None, length=4), + ByteField('table_type', 0), + BitField('reserved3', 0, 4), + BitField('group_type', 0, 4), + ShortField('reserved4', 0), + ByteField('reserved5', 0), + BitField('table_id', 0, 24), + BitField('src_esw_owner_vhca_id_valid', 0, 1), + BitField('reserved6', 0, 31), + IntField('start_flow_index', 0), + BitField('reserved7', 0, 18), + BitField('log2_match_table_size', 0, 6), + BitField('reserved8', 0, 2), + BitField('log2_match_table_rows', 0, 6), + IntField('end_flow_index', 0), + ShortField('reserved9', 0), + ShortField('match_definer_id', 0), + StrFixedLenField('reserved10', None, length=16), + BitField('reserved11', 0, 24), + ByteField('match_criteria_enable', 0), + PacketField('match_criteria', FlowTableEntryMatchParam(), FlowTableEntryMatchParam), + StrFixedLenField('reserved12', None, length=448), + ] + + +class CreateFlowGroupOut(PRMPacket): + fields_desc = [ + ByteField('status', 0), + BitField('reserved1', 0, 24), + IntField('syndrome', 0), + ByteField('reserved2', 0), + BitField('group_id', 0, 24), + StrFixedLenField('reserved3', None, length=4), + ] + + +class DestFormatStruct(PRMPacket): + fields_desc = [ + ByteField('destination_type', 0), + BitField('destination_id', 0, 24), + BitField('dst_esw_owner_vhca_id_valid', 0, 1), + BitField('packet_reformat', 0, 1), + BitField('reserved1', 0, 6), + ByteField('destination_table_type', 0), + BitField('reserved2', 0, 1), + BitField('dst_esw_owner_vhca_id', 0, 15), + ] + + +class ExtendedDestFormat(PRMPacket): + fields_desc = [ + PacketField('extended', DestFormatStruct(), DestFormatStruct), + IntField('packet_reformat_id', 0), + StrFixedLenField('reserved1', None, length=4), + ] + + +class ExtendedFlowCounterList(PRMPacket): + fields_desc = [ + IntField('flow_counter_id', 0), + StrFixedLenField('reserved1', None, length=12), + ] + + +class DestFormatStruct(PRMPacket): + fields_desc = [ + ByteField('destination_type', 0), + BitField('destination_id', 0, 24), + BitField('dst_esw_owner_vhca_id_valid', 0, 1), + BitField('packet_reformat', 0, 1), + BitField('reserved1', 0, 6), + ByteField('destination_table_type', 0), + BitField('reserved2', 0, 1), + BitField('dst_esw_owner_vhca_id', 0, 15), + ] + + +class FlowCounterList(PRMPacket): + fields_desc = [ + IntField('flow_counter_id', 0), + StrFixedLenField('reserved1', None, length=4), + ] + + +class ExeAsoCtrlConnTrack(PRMPacket): + fields_desc = [ + BitField('reserved1', 0, 23), + BitField('direction', 0, 1), + ] + + +class ExeAsoCtrlFlowHit(PRMPacket): + fields_desc = [ + BitField('reserved1', 0, 14), + BitField('read', 0, 1), + BitField('flag_id', 0, 9), + ] + + +class ExeAsoCtrlFlowMeter(PRMPacket): + fields_desc = [ + BitField('reserved1', 0, 20), + BitField('action', 0, 1), + BitField('init_color', 0, 2), + BitField('meter_id', 0, 1), + ] + + +class ExecuteAso(PRMPacket): + fields_desc = [ + BitField('valid', 0, 1), + BitField('reserved1', 0, 7), + BitField('aso_object_id', 0, 24), + BitField('return_reg_id', 0, 4), + BitField('aso_type', 0, 4), + ConditionalField( + PadField(PacketField('flow_meter', ExeAsoCtrlFlowMeter(), ExeAsoCtrlFlowMeter), 4, padwith=b"\x00"), + lambda pkt: pkt.aso_type == 0x2), + ConditionalField( + PadField(PacketField('flow_hit', ExeAsoCtrlFlowHit(), ExeAsoCtrlFlowHit), 4, padwith=b"\x00"), + lambda pkt: pkt.aso_type == 0x4), + ConditionalField( + PadField(PacketField('exe_aso_ctrl', ExeAsoCtrlConnTrack(), ExeAsoCtrlConnTrack), 4, padwith=b"\x00"), + lambda pkt: pkt.aso_type == 0x0), + ] + + +class VlanTag(PRMPacket): + fields_desc = [ + ShortField('tpid', 0), + BitField('pcp', 0, 3), + BitField('dei', 0, 1), + BitField('vid', 0, 12), + ] + + +class FlowContext(PRMPacket): + fields_desc = [ + PacketField('push_vlan_tag', VlanTag(), VlanTag), + IntField('group_id', 0), + ByteField('reserved1', 0), + BitField('flow_tag', 0, 24), + ShortField('reserved2', 0), + ShortField('action', 0), + BitField('extended_destination', 0, 1), + BitField('reserved3', 0, 1), + BitField('flow_source', 0, 2), + BitField('encrypt_decrypt_type', 0, 4), + BitField('destination_list_size', 0, 24), + ByteField('reserved4', 0), + BitField('flow_counter_list_size', 0, 24), + IntField('packet_reformat_id', 0), + PacketField('push_vlan_2_tag', VlanTag(), VlanTag), + IntField('encrypt_decrypt_obj_id', 0), + StrFixedLenField('reserved5', None, length=24), + PacketField('match_value', FlowTableEntryMatchParam(), FlowTableEntryMatchParam), + PacketListField('execute_aso', [ExecuteAso() for x in range(4)], ExecuteAso, count_from=lambda pkt:4), + StrFixedLenField('reserved6', None, length=160), + ConditionalField( + PacketListField('dest_format', [DestFormatStruct() for x in range(0)], DestFormatStruct, count_from=lambda pkt:0), + lambda pkt: pkt.destination_list_size > 0), + ConditionalField( + PacketListField('flow_counter', [FlowCounterList() for x in range(0)], FlowCounterList, count_from=lambda pkt:0), + lambda pkt: pkt.flow_counter_list_size > 0), + ConditionalField( + PacketListField('extended_dest_format', [ExtendedDestFormat() for x in range(0)], ExtendedDestFormat, count_from=lambda pkt:0), + lambda pkt: pkt.destination_list_size > 0), + ConditionalField( + PacketListField('extended_flow_counter', [ExtendedFlowCounterList() for x in range(0)], ExtendedFlowCounterList, count_from=lambda pkt:0), + lambda pkt: pkt.flow_counter_list_size > 0), + ] + + +class SetFlowTableEntryIn(PRMPacket): + fields_desc = [ + ShortField('opcode', DevxOps.MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY), + ShortField('uid', 0), + ShortField('vhca_tunnel_id', 0), + ShortField('op_mod', 0), + BitField('other_vport', 0, 1), + BitField('reserved1', 0, 15), + ShortField('vport_number', 0), + StrFixedLenField('reserved2', None, length=4), + ByteField('table_type', 0), + BitField('reserved3', 0, 24), + ByteField('reserved4', 0), + BitField('table_id', 0, 24), + BitField('ignore_flow_level', 0, 1), + BitField('reserved5', 0, 15), + ShortField('modify_enable_mask', 0), + StrFixedLenField('reserved6', None, length=4), + IntField('flow_index', 0), + StrFixedLenField('reserved7', None, length=28), + PacketField('flow_context', FlowContext(), FlowContext), + ] + + +class SetFlowTableEntryOut(PRMPacket): + fields_desc = [ + ByteField('status', 0), + BitField('reserved1', 0, 24), + IntField('syndrome', 0), + LongField('reserved2', 0) + ] diff --git a/tests/test_mlx5_flow.py b/tests/test_mlx5_flow.py index 0bcd72f16..6a4b8f41f 100644 --- a/tests/test_mlx5_flow.py +++ b/tests/test_mlx5_flow.py @@ -10,18 +10,25 @@ from pyverbs.providers.mlx5.mlx5dv_flow import Mlx5FlowMatcher, \ Mlx5FlowMatcherAttr, Mlx5FlowMatchParameters, Mlx5FlowActionAttr, Mlx5Flow,\ Mlx5PacketReformatFlowAction -from pyverbs.providers.mlx5.mlx5dv import Mlx5Context, Mlx5DVContextAttr +from pyverbs.providers.mlx5.mlx5dv import Mlx5Context, Mlx5DVContextAttr, Mlx5DevxObj from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsUserError -from tests.utils import requires_root_on_eth, PacketConsts import pyverbs.providers.mlx5.mlx5_enums as dve -from tests.mlx5_base import Mlx5RDMATestCase -from tests.base import RawResources import pyverbs.enums as e +from tests.mlx5_base import Mlx5RDMATestCase, create_privileged_context, Mlx5RcResources +from tests.utils import requires_root_on_eth, PacketConsts, is_eth, requires_root +from tests.base import RawResources import tests.utils as u import struct MAX_MATCH_PARAM_SIZE = 0x180 +NIC_RX_RDMA_TABLE_TYPE = 0X7 +NIC_TX_RDMA_TABLE_TYPE = 0X8 +RDMA_TRANSPORT_RX = 0xd +RDMA_TRANSPORT_TX = 0xe +DROP_ACTION = 0X2 +ALLOW_ACTION = 0X1 +TABLE_LEVEL = 200 @u.skip_unsupported @@ -49,8 +56,37 @@ def gen_vxlan_l2_tunnel_encap_header(msg_size): return mac_header + ip_header + udp_header + vxlan_header -class Mlx5FlowResources(RawResources): +def check_rdma_transport_domain_caps(agr_obj): + """ + Check if the device and the given resources support rdma transport domain + creation. If not, it raises unittest.SkipTest. + This function should be called (directly or indirectly) from test functions + only. + :param agr_obj: Aggregation object which contains all resources necessary. + """ + from tests.mlx5_prm_structs import QueryAdvRdmaCapOut, \ + QueryHcaCapIn, QueryCmdHcaCapOut, QueryHcaCapOp, QueryHcaCapMod + query_cap_in = QueryHcaCapIn(op_mod=0x1) + query_cap_out = QueryCmdHcaCapOut(agr_obj.ctx.devx_general_cmd( + query_cap_in, len(QueryCmdHcaCapOut()))) + + if query_cap_out.status: + raise unittest.SkipTest('Failed to query general HCA CAPs with syndrome ' + f'({query_cap_out.syndrome}') + + if not query_cap_out.capability.adv_rdma_cap: + raise unittest.SkipTest("The device doesn't support adv_rdma_cap") + query_adv_rdma_cap_in = QueryHcaCapIn(op_mod=(QueryHcaCapOp.ADV_RDMA_CAP << 0x1) | \ + QueryHcaCapMod.CURRENT) + query_adv_rdma_cap_out = QueryAdvRdmaCapOut(agr_obj.ctx.devx_general_cmd( + query_adv_rdma_cap_in, len(QueryAdvRdmaCapOut()))) + + if not query_adv_rdma_cap_out.capability.rdma_transport_rx_flow_table_properties.ft_support: + raise unittest.SkipTest("The device doesn't support the RDMA transport domain") + + +class Mlx5FlowResources(RawResources): def create_matcher(self, mask, match_criteria_enable, flags=0, ft_type=dve.MLX5DV_FLOW_TABLE_TYPE_NIC_RX_): """ @@ -80,6 +116,121 @@ def create_qps(self): super().create_qps() +class Mlx5RCFlowResources(Mlx5RcResources): + def __init__(self, dev_name, ib_port, gid_index, is_privileged_ctx=False, **kwargs): + """ + Initializes a Mlx5RCFlowResources object with the given values and creates + basic RDMA resources. + :param dev_name: Device name to be used + :param ib_port: IB port of the device to use + :param gid_index: Which GID index to use + :param is_privileged_ctx: If True, creates a privileged context + """ + self.obj_to_cleanup = [] + self.is_privileged_ctx = is_privileged_ctx + super().__init__(dev_name, ib_port, gid_index, **kwargs) + + def create_context(self): + if self.is_privileged_ctx: + create_privileged_context(self) + check_rdma_transport_domain_caps(self) + return + super().create_context() + + def close_resources(self): + for obj in self.obj_to_cleanup: + if obj: + obj.close() + + def create_matcher(self, mask, match_criteria_enable, flags=0, + ft_type=dve.MLX5DV_FLOW_TABLE_TYPE_NIC_RX_, ib_port=1): + """ + Creates a matcher from a provided mask. + :param mask: The mask to match on (in bytes) + :param match_criteria_enable: Bitmask representing which of the + headers and parameters in match_criteria + are used + :param flags: Flow matcher flags + :param ft_type: Flow table type + :param ib_port: Specify its corresponding port. + :return: Resulting matcher + """ + try: + flow_match_param = Mlx5FlowMatchParameters(len(mask), mask) + comp_mask = dve.MLX5DV_FLOW_MATCHER_MASK_FT_TYPE + + if ft_type in [dve.MLX5DV_FLOW_TABLE_TYPE_RDMA_TRANSPORT_RX_, + dve.MLX5DV_FLOW_TABLE_TYPE_RDMA_TRANSPORT_TX_]: + if not is_eth(self.ctx, ib_port): + raise unittest.SkipTest('Must be run on Ethernet link layer') + + comp_mask |= dve.MLX5DV_FLOW_MATCHER_MASK_IB_PORT + + attr = Mlx5FlowMatcherAttr(match_mask=flow_match_param, + match_criteria_enable=match_criteria_enable, + flags=flags, ft_type=ft_type, comp_mask=comp_mask, + ib_port=ib_port) + matcher = Mlx5FlowMatcher(self.ctx, attr) + except PyverbsRDMAError as ex: + if ex.error_code in [errno.EOPNOTSUPP, errno.EPROTONOSUPPORT]: + raise unittest.SkipTest('Matcher creation is not supported') + raise ex + return matcher + + def store_for_cleanup_stage(self, flow_table_obj): + """ + Stores the given object for cleanup in the correct order. + :param flow_table_obj: The object to be stored. + """ + self.obj_to_cleanup.insert(0, flow_table_obj) + + def create_devx_flow_table(self, table_type, level): + """ + Creates a DEVX flow table with the given type and level. + :param table_type: The flow table type. + :param level: The flow table level. + :return: A tuple containing the created flow table object and the table ID. + """ + from tests.mlx5_prm_structs import CreateFlowTableIn,\ + CreateFlowTableOut, FlowTableContext,CreateFlowTableOut + cmd_in = CreateFlowTableIn(table_type=table_type, + flow_table_context=FlowTableContext(level=level)) + flow_table_obj = Mlx5DevxObj(self.ctx, cmd_in, len(CreateFlowTableOut())) + self.store_for_cleanup_stage(flow_table_obj) + out = CreateFlowTableOut(flow_table_obj.out_view) + return flow_table_obj, out.table_id + + def create_devx_flow_group(self, table_id, table_type): + """ + Creates a DEVX flow group for the specified table. + :param table_id: The ID of the flow table. + :param table_type: The type of the flow table. + :return: The ID of the created flow group. + """ + from tests.mlx5_prm_structs import CreateFlowGroupIn, CreateFlowGroupOut + cmd_in = CreateFlowGroupIn(table_type=table_type, + table_id=table_id) + flow_group_ojb = Mlx5DevxObj(self.ctx, cmd_in, len(CreateFlowGroupOut())) + self.store_for_cleanup_stage(flow_group_ojb) + out = CreateFlowGroupOut(flow_group_ojb.out_view) + return out.group_id + + def create_devx_flow_entry(self, table_id, group_id, action, table_type): + """ + Creates a DEVX flow table entry with the specified parameters. + :param table_id: The ID of the flow table. + :param group_id: The ID of the flow group. + :param action: The action for the flow entry. + :param table_type: The type of the flow table. + :return: The created flow table entry object. + """ + from tests.mlx5_prm_structs import SetFlowTableEntryIn, FlowContext, SetFlowTableEntryOut + cmd_in = SetFlowTableEntryIn(table_type=table_type, table_id=table_id, + flow_context=FlowContext(group_id=group_id, action=action)) + flow_table_entry_obj = Mlx5DevxObj(self.ctx, cmd_in, len(SetFlowTableEntryOut())) + self.store_for_cleanup_stage(flow_table_entry_obj) + + class Mlx5MatcherTest(Mlx5RDMATestCase): def setUp(self): super().setUp() @@ -87,6 +238,16 @@ def setUp(self): self.server = None self.client = None + def tearDown(self): + """ + Cleans up resources if manual cleanup is needed. + """ + if self.server and hasattr(self.server, 'obj_to_cleanup'): + self.server.close_resources() + if self.client and hasattr(self.client, 'obj_to_cleanup'): + self.client.close_resources() + super().tearDown() + @u.skip_unsupported def test_create_empty_matcher(self): """ @@ -105,6 +266,58 @@ def test_create_smac_matcher(self): smac_mask = bytes([0xff, 0xff, 0xff, 0xff, 0xff, 0xff]) self.res.create_matcher(smac_mask, u.MatchCriteriaEnable.OUTER) + def generic_test_mlx5_flow_table(self, flow_table_type_mapping, action, ib_port=1, + traffic=False): + """ + This function performs the following steps: + 1. Creates a DEVX flow table with the specified table type. + 2. Creates a flow group in the DEVX flow table. + 3. Inserts a flow table entry (FTE) into the flow table group. + 4. Creates an mlx5 flow with an empty matcher (to steer all packets) on the specified + table type, directing them to the DEVX flow table created in step #1. + 5. If the `traffic` flag is set to `True`, RDMA traffic is run to test the flow steering. + + :param flow_table_type_mapping: Dictionary mapping flow table types. + :param action: Specifies the action to be performed on the matched packets. + :param ib_port: The port for flow matching. + :param traffic: Boolean flag indicating whether RDMA traffic should run. + """ + empty_mask = bytes(MAX_MATCH_PARAM_SIZE) + + for ft_type, table_type in flow_table_type_mapping.items(): + devx_table_obj, table_id = self.server.create_devx_flow_table(table_type, + level=TABLE_LEVEL) + matcher = self.server.create_matcher(empty_mask, u.MatchCriteriaEnable.NONE, + ft_type=ft_type, ib_port=ib_port) + self.server.store_for_cleanup_stage(matcher) + group_id = self.server.create_devx_flow_group(table_id, table_type) + self.server.create_devx_flow_entry(table_id, group_id, action, table_type) + empty_value_param = Mlx5FlowMatchParameters(len(empty_mask), empty_mask) + action_dest = Mlx5FlowActionAttr(action_type=dve.MLX5DV_FLOW_ACTION_DEST_DEVX, + obj=devx_table_obj) + self.server.flow = Mlx5Flow(matcher, empty_value_param, [action_dest], 1) + + if traffic: + u.traffic(client=self.client, server=self.server,iters=self.iters, + gid_idx=self.gid_index, port=self.ib_port, is_cq_ex=True) + + @u.skip_unsupported + @requires_root() + def test_flow_rdma_transport_domain_traffic(self): + """ + Creates a devx table object with the RDMA transport domain, + Verifies that the traffic passes successfully. + """ + self.client = Mlx5RcResources(**self.dev_info) + self.server = Mlx5RCFlowResources(is_privileged_ctx=True, **self.dev_info) + self.pre_run() + self.sync_remote_attr() + flow_table_type_mapping = { + dve.MLX5DV_FLOW_TABLE_TYPE_RDMA_TRANSPORT_RX_: RDMA_TRANSPORT_RX, + dve.MLX5DV_FLOW_TABLE_TYPE_RDMA_TRANSPORT_TX_: RDMA_TRANSPORT_TX} + self.generic_test_mlx5_flow_table(flow_table_type_mapping, action=ALLOW_ACTION, ib_port=1, + traffic=True) + @u.skip_unsupported def test_smac_matcher_to_qp_flow(self): """ From d22198abe1d91e17a9a3ba2322550c8c391b5591 Mon Sep 17 00:00:00 2001 From: dhayon Date: Sun, 22 Dec 2024 17:59:01 +0200 Subject: [PATCH 5/5] tests: Add test to cover RDMA domain Add test that creates DevX objects for RDMA RX and TX domains. This test can be run on IB and ETH. Signed-off-by: Linoy Ganti Signed-off-by: dhayon Signed-off-by: Edward Srouji --- tests/test_mlx5_flow.py | 21 +++++++++++++++++++-- tests/utils.py | 11 +++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/tests/test_mlx5_flow.py b/tests/test_mlx5_flow.py index 6a4b8f41f..fdf87019a 100644 --- a/tests/test_mlx5_flow.py +++ b/tests/test_mlx5_flow.py @@ -15,7 +15,8 @@ import pyverbs.providers.mlx5.mlx5_enums as dve import pyverbs.enums as e from tests.mlx5_base import Mlx5RDMATestCase, create_privileged_context, Mlx5RcResources -from tests.utils import requires_root_on_eth, PacketConsts, is_eth, requires_root +from tests.utils import requires_root_on_eth, PacketConsts, is_eth, requires_root, \ + requires_no_sriov from tests.base import RawResources import tests.utils as u import struct @@ -124,7 +125,7 @@ def __init__(self, dev_name, ib_port, gid_index, is_privileged_ctx=False, **kwar :param dev_name: Device name to be used :param ib_port: IB port of the device to use :param gid_index: Which GID index to use - :param is_privileged_ctx: If True, creates a privileged context + :param is_privileged_ctx: If True, creates a privileged context (default: False) """ self.obj_to_cleanup = [] self.is_privileged_ctx = is_privileged_ctx @@ -301,6 +302,22 @@ def generic_test_mlx5_flow_table(self, flow_table_type_mapping, action, ib_port= u.traffic(client=self.client, server=self.server,iters=self.iters, gid_idx=self.gid_index, port=self.ib_port, is_cq_ex=True) + @u.skip_unsupported + @requires_root() + @requires_no_sriov() + def test_flow_table_drop(self): + """ + Creates rules with DevX objects for RDMA RX and TX tables. + - Creates two flow tables, one for RDMA RX and one for RDMA TX. + - Configures matchers with different flow tables. + - Configures flow actions to drop. + """ + self.create_players(Mlx5RCFlowResources) + flow_table_type_mapping = { + dve.MLX5DV_FLOW_TABLE_TYPE_RDMA_RX_: NIC_RX_RDMA_TABLE_TYPE, + dve.MLX5DV_FLOW_TABLE_TYPE_RDMA_TX_: NIC_TX_RDMA_TABLE_TYPE} + self.generic_test_mlx5_flow_table(flow_table_type_mapping, action=DROP_ACTION) + @u.skip_unsupported @requires_root() def test_flow_rdma_transport_domain_traffic(self): diff --git a/tests/utils.py b/tests/utils.py index bb2444efa..139f91878 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1449,6 +1449,17 @@ def inner(instance): return outer +def requires_no_sriov(): + def outer(func): + def inner(instance): + path = '/sys/class/infiniband/{instance.dev_name}/device/sriov_totalvfs' + if os.path.isfile(path): + raise unittest.SkipTest('Virtual Functions are present on the device') + return func(instance) + return inner + return outer + + def requires_root_on_eth(port_num=1): def outer(func): def inner(instance):