Merge pull request #1734 from ShacharKagan/pyverbs-enhancements-and-fixes

pyverbs: Add WR property getters and fix cleanup errors

Author: rleon

pyverbs/providers/mlx5/dr_action.pxd

@@ -43,6 +43,7 @@ cdef class DrActionPopVLan(DrAction):
 
 cdef class DrActionDestAttr(PyverbsCM):
     cdef DrAction dest
+    cdef DrAction reformat
     cdef dv.mlx5dv_dr_action_dest_attr *action_dest_attr
     cdef dv.mlx5dv_dr_action_dest_reformat *dest_reformat
 

pyverbs/providers/mlx5/dr_action.pyx

@@ -216,14 +216,15 @@ cdef class DrActionDestAttr(PyverbsCM):
         super().__init__()
         self.dest_reformat = NULL
         self.action_dest_attr = NULL
-        if action_type == mlx5dv_dr_action_dest_type.MLX5DV_DR_ACTION_DEST:
-            self.action_dest_attr = <dv.mlx5dv_dr_action_dest_attr *> calloc(
+        self.dest = dest
+        self.reformat = None
+        self.action_dest_attr = <dv.mlx5dv_dr_action_dest_attr *> calloc(
                 1, sizeof(dv.mlx5dv_dr_action_dest_attr))
-            if self.action_dest_attr == NULL:
-                raise PyverbsRDMAErrno('Memory allocation for DrActionDestAttr failed.')
-            self.action_dest_attr.type = action_type
+        if self.action_dest_attr == NULL:
+            raise PyverbsRDMAErrno('Memory allocation for DrActionDestAttr failed.')
+        self.action_dest_attr.type = action_type
+        if action_type == mlx5dv_dr_action_dest_type.MLX5DV_DR_ACTION_DEST:
             self.action_dest_attr.dest = dest.action
-            self.dest = dest
         elif action_type == mlx5dv_dr_action_dest_type.MLX5DV_DR_ACTION_DEST_REFORMAT:
             self.dest_reformat = <dv.mlx5dv_dr_action_dest_reformat *> calloc(
                 1, sizeof(dv.mlx5dv_dr_action_dest_reformat))
@@ -232,6 +233,7 @@ cdef class DrActionDestAttr(PyverbsCM):
             self.action_dest_attr.dest_reformat = self.dest_reformat
             self.action_dest_attr.dest_reformat.reformat = reformat.action
             self.action_dest_attr.dest_reformat.dest = dest.action
+            self.reformat = reformat
         else:
             raise PyverbsError('Unsupported action type is provided.')
 
@@ -248,6 +250,8 @@ cdef class DrActionDestAttr(PyverbsCM):
         if self.dest_reformat != NULL:
             free(self.dest_reformat)
             self.dest_reformat = NULL
+        self.dest = None
+        self.reformat = None
 
 
 cdef class DrActionDestArray(DrAction):
@@ -280,6 +284,11 @@ cdef class DrActionDestArray(DrAction):
             raise PyverbsRDMAErrno('DrActionDestArray creation failed.')
         free(ptr_list)
         domain.dr_actions.add(self)
+        for action in dest_actions:
+            temp_attr = <DrActionDestAttr>action
+            temp_attr.dest.add_ref(self)
+            if temp_attr.reformat:
+                temp_attr.reformat.add_ref(self)
 
     def __dealloc__(self):
         self.close()

pyverbs/providers/mlx5/dr_rule.pxd

@@ -9,3 +9,4 @@ from pyverbs.base cimport PyverbsCM
 cdef class DrRule(PyverbsCM):
     cdef dv.mlx5dv_dr_rule *rule
     cdef object dr_matcher
+    cdef object actions

pyverbs/providers/mlx5/dr_rule.pyx

@@ -35,6 +35,7 @@ cdef class DrRule(PyverbsCM):
             raise PyverbsRDMAErrno('DrRule creation failed.')
         for i in range(0, len(actions)):
             (<DrAction>actions[i]).add_ref(self)
+        self.actions = actions
         matcher.add_ref(self)
         self.dr_matcher = matcher
 
@@ -50,3 +51,4 @@ cdef class DrRule(PyverbsCM):
                 raise PyverbsRDMAError('Failed to destroy DrRule.', rc)
             self.rule = NULL
             self.dr_matcher = None
+            self.actions = None

pyverbs/wr.pyx

@@ -141,6 +141,9 @@ cdef class RecvWR(PyverbsCM):
     def num_sge(self, val):
         self.recv_wr.num_sge = val
 
+    @property
+    def sg_list(self):
+        return <uintptr_t> self.recv_wr.sg_list
 
 cdef class SendWR(PyverbsCM):
     def __init__(self, wr_id=0, opcode=e.IBV_WR_SEND, num_sge=0, imm_data=0,
@@ -238,9 +241,12 @@ cdef class SendWR(PyverbsCM):
     def send_flags(self, val):
         self.send_wr.send_flags = val
 
-    property sg_list:
-        def __set__(self, SGE val not None):
-            self.send_wr.sg_list = val.sge
+    @property
+    def sg_list(self):
+        return <uintptr_t> self.send_wr.sg_list
+    @sg_list.setter
+    def sg_list(self, SGE val not None):
+        self.send_wr.sg_list = val.sge
 
     def set_wr_ud(self, AH ah not None, rqpn, rqkey):
         """
@@ -313,6 +319,14 @@ cdef class SendWR(PyverbsCM):
         """
         self.send_wr.qp_type.xrc.remote_srqn = remote_srqn
 
+    @property
+    def rdma_remote_addr(self):
+        return self.send_wr.wr.rdma.remote_addr
+
+    @property
+    def rdma_rkey(self):
+        return self.send_wr.wr.rdma.rkey
+
 def send_flags_to_str(flags):
     send_flags = {e.IBV_SEND_FENCE: 'IBV_SEND_FENCE',
                   e.IBV_SEND_SIGNALED: 'IBV_SEND_SIGNALED',

tests/test_mlx5_dc.py

@@ -3,6 +3,7 @@
 
 import unittest
 import errno
+import time
 
 from tests.mlx5_base import Mlx5DcResources, Mlx5RDMATestCase, Mlx5DcStreamsRes
 from pyverbs.pyverbs_error import PyverbsRDMAError
@@ -121,9 +122,10 @@ def test_dc_stream_qp_recovery(self):
             with self.assertRaisesRegex(PyverbsRDMAError, r'Remote access error'):
                 u.rdma_traffic(**self.traffic_args, new_send=True,
                                send_op=ibv_wr_opcode.IBV_WR_RDMA_WRITE)
-        # Retry mechanism: QP state update to ERR takes time after errors occur
+        # Poll QP state with timeout: it takes time to transition QP to ERR after errors
         qp_in_err_state = False
-        for _ in range(3):
+        start = time.perf_counter()
+        while time.perf_counter() - start < 1.0:
             qp_attr, _ = self.client.qps[qp_idx].query(ibv_qp_attr_mask.IBV_QP_STATE)
             if qp_attr.cur_qp_state == ibv_qp_state.IBV_QPS_ERR:
                 qp_in_err_state = True