UCT/IB/MLX5: Print KSM details to log

openucx · Jan 14, 2024 · d20602c · d20602c
1 parent b201bd8
commit d20602c
Showing 1 changed file with 36 additions and 11 deletions.
diff --git a/src/uct/ib/mlx5/dv/ib_mlx5dv_md.c b/src/uct/ib/mlx5/dv/ib_mlx5dv_md.c
@@ -89,8 +89,9 @@ uct_ib_mlx5_devx_reg_ksm(uct_ib_mlx5_md_t *md, int atomic, uint64_t address,
                                        out, sizeof(out));
     if (mr == NULL) {
         if (reason != NULL) {
-            ucs_debug("mlx5dv_devx_obj_create(CREATE_MKEY, mode=KSM) failed, "
-                      "syndrome 0x%x: %m",
+            ucs_debug("%s: mlx5dv_devx_obj_create(CREATE_MKEY, mode=KSM, "
+                      "start_addr=0x%lx length=%zu) failed, syndrome 0x%x: %m",
+                      uct_ib_device_name(&md->super.dev), address, length,
                       UCT_IB_MLX5DV_GET(create_mkey_out, out, syndrome));
         }
 
@@ -140,9 +141,16 @@ uct_ib_mlx5_devx_reg_ksm_data_mt(uct_ib_mlx5_md_t *md, int atomic,
     void *mr_address       = address;
     size_t list_size       = ksm_data->mr_num;
     ucs_status_t status;
+    struct ibv_mr **mr;
     char *in;
     void *klm;
-    int i;
+
+    if (reason != NULL) {
+        ucs_trace("%s: initialize ksm_data_mt list for %s address %p length %zu"
+                  " iova 0x%" PRIx64,
+                  uct_ib_device_name(&md->super.dev), reason, address,
+                  ksm_data->length, iova);
+    }
 
     /* Add offset to workaround CREATE_MKEY range check issue */
     if (iova_offset > 0) {
@@ -154,13 +162,18 @@ uct_ib_mlx5_devx_reg_ksm_data_mt(uct_ib_mlx5_md_t *md, int atomic,
         goto out;
     }
 
+    ucs_log_indent(+1);
     klm = UCT_IB_MLX5DV_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
-    for (i = 0; i < ksm_data->mr_num; i++) {
+    ucs_carray_for_each(mr, ksm_data->mrs, ksm_data->mr_num) {
+        ucs_trace("klm[%ld] va %p mr [addr %p len %zu lkey 0x%x]",
+                  mr - ksm_data->mrs, mr_address, (*mr)->addr, (*mr)->length,
+                  (*mr)->lkey);
         UCT_IB_MLX5DV_SET64(klm, klm, address, (uintptr_t)mr_address);
-        UCT_IB_MLX5DV_SET(klm, klm, mkey, ksm_data->mrs[i]->lkey);
+        UCT_IB_MLX5DV_SET(klm, klm, mkey, (*mr)->lkey);
         klm        = UCS_PTR_BYTE_OFFSET(klm, UCT_IB_MLX5DV_ST_SZ_BYTES(klm));
-        mr_address = UCS_PTR_BYTE_OFFSET(mr_address, ksm_data->mrs[i]->length);
+        mr_address = UCS_PTR_BYTE_OFFSET(mr_address, (*mr)->length);
     }
+    ucs_log_indent(-1);
 
     if (iova_offset > 0) {
         UCT_IB_MLX5DV_SET64(klm, klm, address, (uintptr_t)mr_address);
@@ -186,23 +199,35 @@ uct_ib_mlx5_devx_reg_ksm_data_addr(uct_ib_mlx5_md_t *md, struct ibv_mr *mr,
                                    struct mlx5dv_devx_obj **mr_p,
                                    uint32_t *mkey)
 {
-    int i;
-    char *in;
-    void *klm;
     ucs_status_t status;
+    uint64_t mr_address;
+    void *klm;
+    char *in;
+    int i;
+
+    if (reason != NULL) {
+        ucs_trace("%s: initialize ksm_data_addr list for %s address 0x%lx "
+                  "length %zu iova 0x%" PRIx64,
+                  uct_ib_device_name(&md->super.dev), reason, address, length,
+                  iova);
+    }
 
     status = uct_ib_mlx5_alloc_mkey_inbox(list_size, &in);
     if (status != UCS_OK) {
         return status;
     }
 
     klm = UCT_IB_MLX5DV_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
+    ucs_log_indent(+1);
     for (i = 0; i < list_size; i++) {
+        mr_address = address + (i * UCT_IB_MD_MAX_MR_SIZE);
+        ucs_trace("klm[%d] va 0x%lx mr [addr %p len %zu lkey 0x%x]", i,
+                  mr_address, mr->addr, mr->length, mr->lkey);
         UCT_IB_MLX5DV_SET(klm, klm, mkey, mr->lkey);
-        UCT_IB_MLX5DV_SET64(klm, klm, address,
-                            address + (i * UCT_IB_MD_MAX_MR_SIZE));
+        UCT_IB_MLX5DV_SET64(klm, klm, address, mr_address);
         klm = UCS_PTR_BYTE_OFFSET(klm, UCT_IB_MLX5DV_ST_SZ_BYTES(klm));
     }
+    ucs_log_indent(-1);
 
     status = uct_ib_mlx5_devx_reg_ksm(md, atomic, iova, length, list_size,
                                       UCT_IB_MD_MAX_MR_SIZE, in, mkey_index,