From c2f5f0fbc802467b7579ac484227f629a55c4b09 Mon Sep 17 00:00:00 2001 From: randomkang <550941794@qq.com> Date: Mon, 18 May 2026 00:01:42 +0800 Subject: [PATCH 1/2] Add IBV_ACCESS_RELAXED_ORDERING for rdma reg mr --- src/brpc/rdma/rdma_helper.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/brpc/rdma/rdma_helper.cpp b/src/brpc/rdma/rdma_helper.cpp index 9bad33750c..ca5af1f32a 100644 --- a/src/brpc/rdma/rdma_helper.cpp +++ b/src/brpc/rdma/rdma_helper.cpp @@ -65,7 +65,7 @@ int (*IbvQueryQp)(ibv_qp*, ibv_qp_attr*, ibv_qp_attr_mask, ibv_qp_init_attr*) = int (*IbvDestroyQp)(ibv_qp*) = NULL; ibv_comp_channel* (*IbvCreateCompChannel)(ibv_context*) = NULL; int (*IbvDestroyCompChannel)(ibv_comp_channel*) = NULL; -ibv_mr* (*IbvRegMr)(ibv_pd*, void*, size_t, ibv_access_flags) = NULL; +ibv_mr* (*IbvRegMr)(ibv_pd*, void*, size_t, int) = NULL; int (*IbvDeregMr)(ibv_mr*) = NULL; int (*IbvGetCqEvent)(ibv_comp_channel*, ibv_cq**, void**) = NULL; void (*IbvAckCqEvents)(ibv_cq*, unsigned int) = NULL; @@ -176,7 +176,7 @@ void* UserExtendBlockPool(void* region_base, size_t region_size, uint32_t RdmaRegisterMemory(void* buf, size_t size) { // Register the memory as callback in block_pool // The thread-safety should be guaranteed by the caller - ibv_mr* mr = IbvRegMr(g_pd, buf, size, IBV_ACCESS_LOCAL_WRITE); + ibv_mr* mr = IbvRegMr(g_pd, buf, size, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_RELAXED_ORDERING); if (!mr) { PLOG(ERROR) << "Fail to register memory"; return 0; @@ -591,7 +591,7 @@ void GlobalRdmaInitializeOrDie() { } uint32_t RegisterMemoryForRdma(void* buf, size_t len) { - ibv_mr* mr = IbvRegMr(g_pd, buf, len, IBV_ACCESS_LOCAL_WRITE); + ibv_mr* mr = IbvRegMr(g_pd, buf, len, IBV_ACCESS_LOCAL_WRITE |IBV_ACCESS_RELAXED_ORDERING); if (!mr) { PLOG(ERROR) << "Fail to register memory"; return 0; From 9c2fc2f2f029ef395d19c60de6cdf5847d4271e5 Mon Sep 17 00:00:00 2001 From: randomkang <550941794@qq.com> Date: Mon, 18 May 2026 23:49:36 +0800 Subject: [PATCH 2/2] reorg code --- src/brpc/rdma/rdma_helper.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/brpc/rdma/rdma_helper.cpp b/src/brpc/rdma/rdma_helper.cpp index a9233b289f..6a9538f6ab 100644 --- a/src/brpc/rdma/rdma_helper.cpp +++ b/src/brpc/rdma/rdma_helper.cpp @@ -180,8 +180,12 @@ uint32_t RdmaRegisterMemory(void* buf, size_t size) { // The thread-safety should be guaranteed by the caller ibv_mr* mr = IbvRegMr(g_pd, buf, size, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_RELAXED_ORDERING); if (!mr) { - PLOG(ERROR) << "Fail to register memory"; - return 0; + PLOG(WARNING) << "Do not support IBV_ACCESS_RELAXED_ORDERING for RDMA!!!"; + mr = IbvRegMr(g_pd, buf, size, IBV_ACCESS_LOCAL_WRITE); + if (!mr) { + PLOG(ERROR) << "Fail to register memory"; + return 0; + } } g_mrs->push_back(mr); return mr->lkey; @@ -596,8 +600,12 @@ void GlobalRdmaInitializeOrDie() { uint32_t RegisterMemoryForRdma(void* buf, size_t len) { ibv_mr* mr = IbvRegMr(g_pd, buf, len, IBV_ACCESS_LOCAL_WRITE |IBV_ACCESS_RELAXED_ORDERING); if (!mr) { - PLOG(ERROR) << "Fail to register memory"; - return 0; + PLOG(WARNING) << "Do not support IBV_ACCESS_RELAXED_ORDERING for RDMA!!!"; + mr = IbvRegMr(g_pd, buf, len, IBV_ACCESS_LOCAL_WRITE); + if (!mr) { + PLOG(ERROR) << "Fail to register memory"; + return 0; + } } { BAIDU_SCOPED_LOCK(*g_user_mrs_lock);