diff --git a/0004-update-ZTE-Dinghai-RDMA-driver.patch b/0004-update-ZTE-Dinghai-RDMA-driver.patch new file mode 100644 index 0000000000000000000000000000000000000000..607ae8deb8cbc8bf3c4f021e5f042aae22c364e0 --- /dev/null +++ b/0004-update-ZTE-Dinghai-RDMA-driver.patch @@ -0,0 +1,14376 @@ +diff -ruN baseline/providers/zrdma/CMakeLists.txt b/providers/zrdma/CMakeLists.txt +--- a/providers/zrdma/CMakeLists.txt 2026-08-06 15:37:55.337861733 +0800 ++++ b/providers/zrdma/CMakeLists.txt 2026-08-06 15:29:37.362683612 +0800 +@@ -1,17 +1,26 @@ + # SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */ + # Copyright (c) 2024 ZTE Corporation. All rights reserved. */ + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror") ++if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64") ++ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx -mavx2") ++endif() ++RDMA_AddOptLDFlag(CMAKE_SHARED_LINKER_FLAGS SUPPORTS_BUILD_ID "-Wl,--build-id=uuid") + rdma_shared_provider(zrdma libzrdma.map + 1 1.1.${PACKAGE_VERSION} + zxdh_hw.c +- main.c ++ zxdh_zrdma.c + zxdh_verbs.c + private_verbs_cmd.c ++ zxdh_devx.c ++ wqe_rate_limit.c + ) ++include_directories(${NL_INCLUDE_DIRS}) + + publish_headers(infiniband + zxdh_dv.h ++ zxdh_devids.h + ) + ++target_link_libraries(zrdma PRIVATE nl-3) + + rdma_pkg_config("zrdma" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}") +diff -ruN baseline/providers/zrdma/libzrdma.map b/providers/zrdma/libzrdma.map +--- a/providers/zrdma/libzrdma.map 2026-08-06 15:37:55.337861733 +0800 ++++ b/providers/zrdma/libzrdma.map 2026-08-06 15:29:37.485692060 +0800 +@@ -13,4 +13,34 @@ + zxdh_query_qpc; + zxdh_modify_qpc; + zxdh_reset_qp; ++ zxdh_cap_start; ++ zxdh_cap_stop; ++ zxdh_cap_free; ++ zxdh_mp_capture; ++ zxdh_mp_get_data; ++ zxdh_mp_capture_clear; ++ zxdh_get_active_vhca_gqps; ++ zxdh_rdma_hmc_query; ++ zxdh_get_object_data; ++ zxdh_get_cc_basic_info; ++ zxdh_rdma_health_check; ++ zxdh_cfg_dev_parameter; ++ zxdh_show_res_map; ++ zxdh_read_ram; ++ zxdh_devx_get_qp_init_size; ++ zxdh_devx_create_qp; ++ zxdh_devx_destroy_qp; ++ zxdh_devx_modify_qp; ++ zxdh_devx_get_cq_init_size; ++ zxdh_devx_create_cq; ++ zxdh_devx_destroy_cq; ++ zxdh_devx_modify_cq; ++ zxdh_rdma_switch_query; ++ zxdh_rdma_switch_config; ++ zxdh_devx_reg_mr; ++ zxdh_devx_dereg_mr; ++ zxdh_devx_create_cq_with_ext_buf; ++ zxdh_devx_create_qp_with_ext_buf; ++ zxdh_devx_get_peer_dev_sbdf; ++ zxdh_mp_capture_gqp; + } ZRDMA_1.0; +diff -ruN baseline/providers/zrdma/main.c b/providers/zrdma/main.c +--- a/providers/zrdma/main.c 2026-08-06 15:37:55.337861733 +0800 ++++ b/providers/zrdma/main.c 1970-01-01 08:00:00.000000000 +0800 +@@ -1,202 +0,0 @@ +-// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB +-/* Copyright (c) 2024 ZTE Corporation. All rights reserved. */ +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include "zxdh_devids.h" +-#include "main.h" +-#include "zxdh_abi.h" +-#include "private_verbs_cmd.h" +- +-#define ZXDH_HCA(v, d) VERBS_PCI_MATCH(v, d, NULL) +-static const struct verbs_match_ent hca_table[] = { +- VERBS_DRIVER_ID(RDMA_DRIVER_ZXDH), +- ZXDH_HCA(PCI_VENDOR_ID_ZXDH_EVB, ZXDH_DEV_ID_ADAPTIVE_EVB_PF), +- ZXDH_HCA(PCI_VENDOR_ID_ZXDH_EVB, ZXDH_DEV_ID_ADAPTIVE_EVB_VF), +- ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312, ZXDH_DEV_ID_ADAPTIVE_E312_PF), +- ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312, ZXDH_DEV_ID_ADAPTIVE_E312_VF), +- ZXDH_HCA(PCI_VENDOR_ID_ZXDH_X512, ZXDH_DEV_ID_ADAPTIVE_X512_PF), +- ZXDH_HCA(PCI_VENDOR_ID_ZXDH_X512, ZXDH_DEV_ID_ADAPTIVE_X512_VF), +- {} +-}; +- +-/** +- * zxdh_ufree_context - free context that was allocated +- * @ibctx: context allocated ptr +- */ +-static void zxdh_ufree_context(struct ibv_context *ibctx) +-{ +- struct zxdh_uvcontext *iwvctx; +- +- iwvctx = container_of(ibctx, struct zxdh_uvcontext, ibv_ctx.context); +- +- zxdh_ufree_pd(&iwvctx->iwupd->ibv_pd); +- zxdh_munmap(iwvctx->sq_db); +- zxdh_munmap(iwvctx->cq_db); +- verbs_uninit_context(&iwvctx->ibv_ctx); +- free(iwvctx); +-} +- +-static const struct verbs_context_ops zxdh_uctx_ops = { +- .alloc_mw = zxdh_ualloc_mw, +- .alloc_pd = zxdh_ualloc_pd, +- .attach_mcast = zxdh_uattach_mcast, +- .bind_mw = zxdh_ubind_mw, +- .cq_event = zxdh_cq_event, +- .create_ah = zxdh_ucreate_ah, +- .create_cq = zxdh_ucreate_cq, +- .create_cq_ex = zxdh_ucreate_cq_ex, +- .create_qp = zxdh_ucreate_qp, +- .create_qp_ex = zxdh_ucreate_qp_ex, +- .create_srq = zxdh_ucreate_srq, +- .dealloc_mw = zxdh_udealloc_mw, +- .dealloc_pd = zxdh_ufree_pd, +- .dereg_mr = zxdh_udereg_mr, +- .destroy_ah = zxdh_udestroy_ah, +- .destroy_cq = zxdh_udestroy_cq, +- .modify_cq = zxdh_umodify_cq, +- .destroy_qp = zxdh_udestroy_qp, +- .destroy_srq = zxdh_udestroy_srq, +- .detach_mcast = zxdh_udetach_mcast, +- .modify_qp = zxdh_umodify_qp, +- .modify_srq = zxdh_umodify_srq, +- .poll_cq = zxdh_upoll_cq, +- .post_recv = zxdh_upost_recv, +- .post_send = zxdh_upost_send, +- .post_srq_recv = zxdh_upost_srq_recv, +- .query_device_ex = zxdh_uquery_device_ex, +- .query_port = zxdh_uquery_port, +- .query_qp = zxdh_uquery_qp, +- .query_srq = zxdh_uquery_srq, +- .reg_mr = zxdh_ureg_mr, +- .rereg_mr = zxdh_urereg_mr, +- .req_notify_cq = zxdh_uarm_cq, +- .resize_cq = zxdh_uresize_cq, +- .free_context = zxdh_ufree_context, +- .get_srq_num = zxdh_uget_srq_num, +-}; +- +-/** +- * zxdh_ualloc_context - allocate context for user app +- * @ibdev: ib device created during zxdh_driver_init +- * @cmd_fd: save fd for the device +- * @private_data: device private data +- * +- * Returns callback routine table and calls driver for allocating +- * context and getting back resource information to return as ibv_context. +- */ +-static struct verbs_context *zxdh_ualloc_context(struct ibv_device *ibdev, +- int cmd_fd, void *private_data) +-{ +- struct ibv_pd *ibv_pd; +- struct zxdh_uvcontext *iwvctx; +- struct zxdh_get_context cmd; +- struct zxdh_get_context_resp resp = {}; +- __u64 sq_db_mmap_key, cq_db_mmap_key; +- __u8 user_ver = ZXDH_ABI_VER; +- +- iwvctx = verbs_init_and_alloc_context(ibdev, cmd_fd, iwvctx, ibv_ctx, +- RDMA_DRIVER_ZXDH); +- if (!iwvctx) +- return NULL; +- +- zxdh_set_debug_mask(); +- iwvctx->zxdh_write_imm_split_switch = zxdh_get_write_imm_split_switch(); +- cmd.userspace_ver = user_ver; +- if (ibv_cmd_get_context(&iwvctx->ibv_ctx, +- (struct ibv_get_context *)&cmd, sizeof(cmd), +- &resp.ibv_resp, sizeof(resp))) { +- cmd.userspace_ver = 4; +- if (ibv_cmd_get_context( +- &iwvctx->ibv_ctx, (struct ibv_get_context *)&cmd, +- sizeof(cmd), &resp.ibv_resp, sizeof(resp))) +- goto err_free; +- user_ver = cmd.userspace_ver; +- } +- +- verbs_set_ops(&iwvctx->ibv_ctx, &zxdh_uctx_ops); +- +- iwvctx->dev_attrs.feature_flags = resp.feature_flags; +- iwvctx->dev_attrs.hw_rev = resp.hw_rev; +- iwvctx->dev_attrs.max_hw_wq_frags = resp.max_hw_wq_frags; +- iwvctx->dev_attrs.max_hw_read_sges = resp.max_hw_read_sges; +- iwvctx->dev_attrs.max_hw_inline = resp.max_hw_inline; +- iwvctx->dev_attrs.max_hw_rq_quanta = resp.max_hw_rq_quanta; +- iwvctx->dev_attrs.max_hw_srq_quanta = resp.max_hw_srq_quanta; +- iwvctx->dev_attrs.max_hw_wq_quanta = resp.max_hw_wq_quanta; +- iwvctx->dev_attrs.max_hw_srq_wr = resp.max_hw_srq_wr; +- iwvctx->dev_attrs.max_hw_sq_chunk = resp.max_hw_sq_chunk; +- iwvctx->dev_attrs.max_hw_cq_size = resp.max_hw_cq_size; +- iwvctx->dev_attrs.min_hw_cq_size = resp.min_hw_cq_size; +- iwvctx->abi_ver = user_ver; +- +- sq_db_mmap_key = resp.sq_db_mmap_key; +- cq_db_mmap_key = resp.cq_db_mmap_key; +- +- iwvctx->dev_attrs.db_addr_type = resp.db_addr_type; +- +- iwvctx->sq_db = zxdh_mmap(cmd_fd, sq_db_mmap_key); +- if (iwvctx->sq_db == MAP_FAILED) +- goto err_free; +- +- iwvctx->cq_db = zxdh_mmap(cmd_fd, cq_db_mmap_key); +- if (iwvctx->cq_db == MAP_FAILED) { +- zxdh_munmap(iwvctx->sq_db); +- goto err_free; +- } +- ibv_pd = zxdh_ualloc_pd(&iwvctx->ibv_ctx.context); +- if (!ibv_pd) { +- zxdh_munmap(iwvctx->sq_db); +- zxdh_munmap(iwvctx->cq_db); +- goto err_free; +- } +- +- ibv_pd->context = &iwvctx->ibv_ctx.context; +- iwvctx->iwupd = container_of(ibv_pd, struct zxdh_upd, ibv_pd); +- add_private_ops(iwvctx); +- return &iwvctx->ibv_ctx; +- +-err_free: +- free(iwvctx); +- +- return NULL; +-} +- +-static void zxdh_uninit_device(struct verbs_device *verbs_device) +-{ +- struct zxdh_udevice *dev; +- +- dev = container_of(&verbs_device->device, struct zxdh_udevice, +- ibv_dev.device); +- free(dev); +-} +- +-static struct verbs_device *zxdh_device_alloc(struct verbs_sysfs_dev *sysfs_dev) +-{ +- struct zxdh_udevice *dev; +- +- dev = calloc(1, sizeof(*dev)); +- if (!dev) +- return NULL; +- +- return &dev->ibv_dev; +-} +- +-static const struct verbs_device_ops zxdh_udev_ops = { +- .alloc_context = zxdh_ualloc_context, +- .alloc_device = zxdh_device_alloc, +- .match_max_abi_version = ZXDH_MAX_ABI_VERSION, +- .match_min_abi_version = ZXDH_MIN_ABI_VERSION, +- .match_table = hca_table, +- .name = "zxdh", +- .uninit_device = zxdh_uninit_device, +-}; +- +-PROVIDER_DRIVER(zxdh, zxdh_udev_ops); +diff -ruN baseline/providers/zrdma/main.h b/providers/zrdma/main.h +--- a/providers/zrdma/main.h 2026-08-06 15:37:55.338861801 +0800 ++++ b/providers/zrdma/main.h 1970-01-01 08:00:00.000000000 +0800 +@@ -1,223 +0,0 @@ +-/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */ +-/* Copyright (c) 2024 ZTE Corporation. All rights reserved. */ +-#ifndef ZXDH_UMAIN_H +-#define ZXDH_UMAIN_H +- +-#include +-#include +-#include +-#include +-#include +- +-#include "zxdh_defs.h" +-#include "zxdh_status.h" +-#include "zxdh_verbs.h" +- +-#define ZXDH_BASE_PUSH_PAGE 1 +-#define ZXDH_U_MINCQ_SIZE 4 +-#define ZXDH_DB_SHADOW_AREA_SIZE 8 +-#define ZXDH_DB_SQ_OFFSET 0x404 +-#define ZXDH_DB_CQ_OFFSET 0x588 +- +-#define MIN_UDP_SPORT 1024 +-#define MIN_QP_QPN 1 +- +-enum zxdh_supported_wc_flags { +- ZXDH_CQ_SUPPORTED_WC_FLAGS = +- IBV_WC_EX_WITH_BYTE_LEN | IBV_WC_EX_WITH_IMM | +- IBV_WC_EX_WITH_QP_NUM | IBV_WC_EX_WITH_SRC_QP | +- IBV_WC_EX_WITH_SLID | IBV_WC_EX_WITH_SL | +- IBV_WC_EX_WITH_DLID_PATH_BITS | +- IBV_WC_EX_WITH_COMPLETION_TIMESTAMP_WALLCLOCK | +- IBV_WC_EX_WITH_COMPLETION_TIMESTAMP, +-}; +- +-enum { +- ZXDH_DBG_QP = 1 << 0, +- ZXDH_DBG_CQ = 1 << 1, +- ZXDH_DBG_SRQ = 1 << 2, +-}; +-extern uint32_t zxdh_debug_mask; +-#define zxdh_dbg(ctx, mask, format, arg...) \ +- do { \ +- if (mask & zxdh_debug_mask) { \ +- int zxdh_dbg_tmp = errno; \ +- verbs_debug(ctx, format, ##arg); \ +- errno = zxdh_dbg_tmp; \ +- } \ +- } while (0) +- +-struct zxdh_udevice { +- struct verbs_device ibv_dev; +-}; +- +-struct zxdh_uah { +- struct ibv_ah ibv_ah; +- uint32_t ah_id; +- struct ibv_global_route grh; +-}; +- +-struct zxdh_upd { +- struct ibv_pd ibv_pd; +- void *arm_cq_page; +- void *arm_cq; +- uint32_t pd_id; +-}; +- +-struct zxdh_uvcontext { +- struct verbs_context ibv_ctx; +- struct zxdh_upd *iwupd; +- struct zxdh_dev_attrs dev_attrs; +- void *db; +- void *sq_db; +- void *cq_db; +- int abi_ver; +- bool legacy_mode; +- uint8_t zxdh_write_imm_split_switch; +- struct zxdh_uvcontext_ops *cxt_ops; +-}; +- +-struct zxdh_uqp; +- +-struct zxdh_cq_buf { +- struct list_node list; +- struct zxdh_cq cq; +- struct verbs_mr vmr; +-}; +- +-struct zxdh_ucq { +- struct verbs_cq verbs_cq; +- struct verbs_mr vmr; +- struct verbs_mr vmr_shadow_area; +- pthread_spinlock_t lock; +- size_t buf_size; +- bool is_armed; +- enum zxdh_cmpl_notify last_notify; +- int comp_vector; +- uint32_t report_rtt; +- struct zxdh_uqp *uqp; +- struct zxdh_cq cq; +- struct list_head resize_list; +- /* for extended CQ completion fields */ +- struct zxdh_cq_poll_info cur_cqe; +- bool resize_enable; +-}; +- +-struct zxdh_usrq { +- struct ibv_srq ibv_srq; +- struct verbs_mr vmr; +- struct verbs_mr list_vmr; +- struct verbs_mr db_vmr; +- size_t total_buf_size; +- size_t buf_size; +- size_t list_buf_size; +- size_t db_buf_size; +- size_t srq_size; +- size_t srq_list_size; +- uint32_t srq_id; +- uint32_t max_wr; +- uint32_t max_sge; +- uint32_t srq_limit; +- pthread_spinlock_t lock; +- uint32_t wq_size; +- struct ibv_recv_wr *pend_rx_wr; +- struct zxdh_srq srq; +-}; +- +-struct zxdh_uqp { +- struct verbs_qp vqp; +- struct zxdh_ucq *send_cq; +- struct zxdh_ucq *recv_cq; +- struct zxdh_usrq *srq; +- struct verbs_mr vmr; +- size_t buf_size; +- uint32_t zxdh_drv_opt; +- pthread_spinlock_t lock; +- uint16_t sq_sig_all; +- uint16_t qperr; +- uint16_t rsvd; +- uint32_t pending_rcvs; +- uint32_t wq_size; +- struct ibv_recv_wr *pend_rx_wr; +- struct zxdh_qp qp; +- enum ibv_qp_type qp_type; +- struct zxdh_sge *recv_sges; +- uint8_t is_srq; +- uint8_t inline_data[ZXDH_MAX_INLINE_DATA_SIZE]; +-}; +- +-struct zxdh_umr { +- struct verbs_mr vmr; +- uint32_t acc_flags; +- uint8_t leaf_pbl_size; +- uint8_t host_page_size; +- uint64_t mr_pa_pble_index; +-}; +- +-/* zxdh_verbs.c */ +-int zxdh_uquery_device_ex(struct ibv_context *context, +- const struct ibv_query_device_ex_input *input, +- struct ibv_device_attr_ex *attr, size_t attr_size); +-int zxdh_uquery_port(struct ibv_context *context, uint8_t port, +- struct ibv_port_attr *attr); +-struct ibv_pd *zxdh_ualloc_pd(struct ibv_context *context); +-int zxdh_ufree_pd(struct ibv_pd *pd); +-struct ibv_mr *zxdh_ureg_mr(struct ibv_pd *pd, void *addr, size_t length, +- uint64_t hca_va, int access); +-int zxdh_udereg_mr(struct verbs_mr *vmr); +- +-int zxdh_urereg_mr(struct verbs_mr *mr, int flags, struct ibv_pd *pd, +- void *addr, size_t length, int access); +- +-struct ibv_mw *zxdh_ualloc_mw(struct ibv_pd *pd, enum ibv_mw_type type); +-int zxdh_ubind_mw(struct ibv_qp *qp, struct ibv_mw *mw, +- struct ibv_mw_bind *mw_bind); +-int zxdh_udealloc_mw(struct ibv_mw *mw); +-struct ibv_cq *zxdh_ucreate_cq(struct ibv_context *context, int cqe, +- struct ibv_comp_channel *channel, +- int comp_vector); +-struct ibv_cq_ex *zxdh_ucreate_cq_ex(struct ibv_context *context, +- struct ibv_cq_init_attr_ex *attr_ex); +-void zxdh_ibvcq_ex_fill_priv_funcs(struct zxdh_ucq *iwucq, +- struct ibv_cq_init_attr_ex *attr_ex); +-int zxdh_uresize_cq(struct ibv_cq *cq, int cqe); +-int zxdh_udestroy_cq(struct ibv_cq *cq); +-int zxdh_umodify_cq(struct ibv_cq *cq, struct ibv_modify_cq_attr *attr); +-int zxdh_upoll_cq(struct ibv_cq *cq, int entries, struct ibv_wc *entry); +-int zxdh_uarm_cq(struct ibv_cq *cq, int solicited); +-void zxdh_cq_event(struct ibv_cq *cq); +-struct ibv_qp *zxdh_ucreate_qp(struct ibv_pd *pd, +- struct ibv_qp_init_attr *attr); +-struct ibv_qp *zxdh_ucreate_qp_ex(struct ibv_context *context, +- struct ibv_qp_init_attr_ex *attr); +-int zxdh_uquery_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask, +- struct ibv_qp_init_attr *init_attr); +-int zxdh_umodify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask); +-int zxdh_udestroy_qp(struct ibv_qp *qp); +-int zxdh_upost_send(struct ibv_qp *ib_qp, struct ibv_send_wr *ib_wr, +- struct ibv_send_wr **bad_wr); +-int zxdh_upost_recv(struct ibv_qp *ib_qp, struct ibv_recv_wr *ib_wr, +- struct ibv_recv_wr **bad_wr); +-struct ibv_srq *zxdh_ucreate_srq(struct ibv_pd *pd, +- struct ibv_srq_init_attr *srq_init_attr); +-int zxdh_udestroy_srq(struct ibv_srq *srq); +-int zxdh_umodify_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr, +- int srq_attr_mask); +-int zxdh_uquery_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr); +-int zxdh_upost_srq_recv(struct ibv_srq *srq, struct ibv_recv_wr *recv_wr, +- struct ibv_recv_wr **bad_recv_wr); +-int zxdh_uget_srq_num(struct ibv_srq *srq, uint32_t *srq_num); +-struct ibv_ah *zxdh_ucreate_ah(struct ibv_pd *ibpd, struct ibv_ah_attr *attr); +-int zxdh_udestroy_ah(struct ibv_ah *ibah); +-int zxdh_uattach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, +- uint16_t lid); +-int zxdh_udetach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, +- uint16_t lid); +-void zxdh_async_event(struct ibv_context *context, +- struct ibv_async_event *event); +-void zxdh_set_hw_attrs(struct zxdh_hw_attrs *attrs); +-void *zxdh_mmap(int fd, off_t offset); +-void zxdh_munmap(void *map); +-void zxdh_set_debug_mask(void); +-int zxdh_get_write_imm_split_switch(void); +-#endif /* ZXDH_UMAIN_H */ +diff -ruN baseline/providers/zrdma/private_verbs_cmd.c b/providers/zrdma/private_verbs_cmd.c +--- a/providers/zrdma/private_verbs_cmd.c 2026-08-06 15:37:55.338861801 +0800 ++++ b/providers/zrdma/private_verbs_cmd.c 2026-08-06 15:29:37.141668434 +0800 +@@ -3,7 +3,15 @@ + #include + #include + #include "private_verbs_cmd.h" +-#include "zxdh_dv.h" ++#include "zxdh_devx.h" ++ ++static inline uint32_t get_log2(uint32_t x) ++{ ++ uint32_t r = 0; ++ for (x >>= 1; x > 0; x >>= 1) ++ r++; ++ return r; ++} + + static void copy_query_qpc(struct zxdh_query_qpc_resp *resp, + struct zxdh_rdma_qpc *qpc) +@@ -64,7 +72,6 @@ + { + DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_QP_OBJ, + ZXDH_IB_METHOD_QP_RESET_QP, 2); +- + fill_attr_in_obj(cmd, ZXDH_IB_ATTR_QP_RESET_QP_HANDLE, qp->handle); + fill_attr_in_uint64(cmd, ZXDH_IB_ATTR_QP_RESET_OP_CODE, opcode); + return execute_ioctl(qp->context, cmd); +@@ -119,6 +126,245 @@ + return execute_ioctl(ibctx, cmd); + } + ++static int _zxdh_cap_start(struct ibv_context *ibctx, ++ struct zxdh_cap_cfg *cap_cfg, ++ struct zxdh_rdma_cap_pa *cap_pa) ++{ ++ int ret; ++ struct zxdh_cap_start_resp resp = { 0 }; ++ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEV, ++ ZXDH_IB_METHOD_DEV_CAP_START, 2); ++ fill_attr_in_ptr(cmd, ZXDH_IB_ATTR_DEV_CAP_START, cap_cfg); ++ fill_attr_out_ptr(cmd, ZXDH_IB_ATTR_DEV_CAP_START_RESP, &resp); ++ ret = execute_ioctl(ibctx, cmd); ++ if (ret == 0) { ++ cap_pa->cap_pa_node0 = resp.cap_pa_node0; ++ cap_pa->cap_pa_node1 = resp.cap_pa_node1; ++ } else { ++ printf("execute_ioctl failed,%d\n", ret); ++ } ++ ++ return ret; ++} ++ ++static int _zxdh_cap_stop(struct ibv_context *ibctx, uint8_t param) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEV, ++ ZXDH_IB_METHOD_DEV_CAP_STOP, 1); ++ fill_attr_in(cmd, ZXDH_IB_ATTR_DEV_CAP_STOP, ¶m, sizeof(param)); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_cap_free(struct ibv_context *ibctx, uint8_t param) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEV, ++ ZXDH_IB_METHOD_DEV_CAP_FREE, 1); ++ fill_attr_in(cmd, ZXDH_IB_ATTR_DEV_CAP_FREE, ¶m, sizeof(param)); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_mp_capture(struct ibv_context *ibctx, ++ struct zxdh_mp_cap_cfg *cap_mp_cfg, ++ struct zxdh_mp_cap_gqp *cap_mp_gqp) ++{ ++ struct zxdh_mp_cap_resp resp = { 0 }; ++ int ret; ++ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEV, ++ ZXDH_IB_METHOD_DEV_MP_CAP, 2); ++ ++ fill_attr_in_ptr(cmd, ZXDH_IB_ATTR_DEV_MP_CAP, cap_mp_cfg); ++ fill_attr_out_ptr(cmd, ZXDH_IB_ATTR_DEV_MP_CAP_RESP, &resp); ++ ret = execute_ioctl(ibctx, cmd); ++ ++ if (ret == 0) { ++ cap_mp_gqp->mcode_type = resp.mcode_type; ++ cap_mp_gqp->cap_pa = resp.cap_pa; ++ cap_mp_gqp->gqp_num = resp.cap_gqp_num; ++ memcpy(cap_mp_gqp->gqpid, resp.cap_gqpid, ++ sizeof(cap_mp_gqp->gqpid)); ++ } else { ++ printf("execute_ioctl for mp capture failed,%d\n", ret); ++ } ++ ++ return ret; ++} ++ ++static int _zxdh_mp_capture_gqp(struct ibv_context *ibctx, ++ struct zxdh_mp_cap_gqp_cfg *cfg, ++ struct zxdh_mp_cap_gqp *cap_mp_gqp) ++{ ++ struct zxdh_mp_cap_resp resp = { 0 }; ++ int ret; ++ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEVICE_PRIV, ++ ZXDH_IB_METHOD_DEV_MP_CAP_GQP, 2); ++ ++ fill_attr_in_ptr(cmd, ZXDH_IB_ATTR_DEV_MP_CAP_GQP, cfg); ++ fill_attr_out_ptr(cmd, ZXDH_IB_ATTR_DEV_MP_CAP_GQP_RESP, &resp); ++ ret = execute_ioctl(ibctx, cmd); ++ if (ret) { ++ printf("execute_ioctl for mp capture by gqp failed,%d\n", ret); ++ return ret; ++ } ++ ++ cap_mp_gqp->mcode_type = resp.mcode_type; ++ cap_mp_gqp->cap_pa = resp.cap_pa; ++ cap_mp_gqp->gqp_num = resp.cap_gqp_num; ++ memcpy(cap_mp_gqp->gqpid, resp.cap_gqpid, ++ sizeof(cap_mp_gqp->gqpid)); ++ ++ return 0; ++} ++ ++static int _zxdh_mp_get_data(struct ibv_context *ibctx, uint8_t param) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEV, ++ ZXDH_IB_METHOD_DEV_MP_GET_DATA, 1); ++ ++ fill_attr_in(cmd, ZXDH_IB_ATTR_DEV_MP_GET_DATA, ¶m, sizeof(param)); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_mp_capture_clear(struct ibv_context *ibctx, ++ struct zxdh_cap_gqp *cap_gqp) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEV, ++ ZXDH_IB_METHOD_DEV_MP_CAP_CLEAR, 1); ++ ++ fill_attr_in_ptr(cmd, ZXDH_IB_ATTR_DEV_MP_CAP_CLEAR, cap_gqp); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_get_active_vhca_gqps(struct ibv_context *ibctx, ++ struct zxdh_active_vhca_gqps *resp) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEVICE_EX, ++ ZXDH_IB_METHOD_DEV_GET_ACT_VHCA_GQPS, 1); ++ ++ fill_attr_out_ptr(cmd, ZXDH_IB_ATTR_DEV_GET_ACT_VHCA_GQPS_RESP, resp); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_rdma_hmc_query(struct ibv_context *ibctx, ++ struct zxdh_context_req *req, ++ struct zxdh_context_resp *resp) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEVICE_EX, ++ ZXDH_IB_METHOD_DEV_GET_HMC, 2); ++ fill_attr_in_ptr(cmd, ZXDH_IB_ATTR_DEV_GET_HMC, req); ++ fill_attr_out_ptr(cmd, ZXDH_IB_ATTR_DEV_GET_HMC_RESP, resp); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_rdma_switch_query(struct ibv_context *ibctx, ++ struct query_rdma_switch_req *req, ++ struct query_rdma_switch_resp *resp) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEVICE_OBJ, ++ ZXDH_IB_METHOD_DEV_QUERY_RDMA_SWITCH, 2); ++ fill_attr_in_ptr(cmd, ZXDH_IB_ATTR_DEV_GET_SWITCH, req); ++ fill_attr_out_ptr(cmd, ZXDH_IB_ATTR_DEV_GET_SWITCH_RESP, resp); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_rdma_switch_config(struct ibv_context *ibctx, ++ struct config_rdma_switch_req *req) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEVICE_OBJ, ++ ZXDH_IB_METHOD_DEV_CONFIG_RDMA_SWITCH, 1); ++ fill_attr_in_ptr(cmd, ZXDH_IB_ATTR_DEV_CONFIG_SWITCH, req); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_get_object_data(struct ibv_context *ibctx, ++ struct zxdh_get_object_data_req *req, ++ struct zxdh_get_object_data_resp *resp) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEVICE_EX, ++ ZXDH_IB_METHOD_DEV_GET_OBJ_DATA, 2); ++ fill_attr_in_ptr(cmd, ZXDH_IB_ATTR_DEV_GET_OBJ_DATA, req); ++ fill_attr_out_ptr(cmd, ZXDH_IB_ATTR_DEV_GET_OBJ_DATA_RESP, resp); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_get_cc_basic_info(struct ibv_context *ibctx, ++ struct zxdh_cc_basic_info *resp) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEVICE_EX, ++ ZXDH_IB_METHOD_DEV_GET_CC_BASIC_INFO, 1); ++ ++ fill_attr_out_ptr(cmd, ZXDH_IB_ATTR_DEV_GET_CC_BASIC_INFO_RESP, resp); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_rdma_health_check(struct ibv_context *ibctx, ++ struct zxdh_health_check_req *req, ++ struct zxdh_health_check_resp *resp) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEVICE_EX, ++ ZXDH_IB_METHOD_DEV_HEALTH_CHECK, 2); ++ fill_attr_in_ptr(cmd, ZXDH_IB_ATTR_DEV_HEALTH_CHECK, req); ++ fill_attr_out_ptr(cmd, ZXDH_IB_ATTR_DEV_HEALTH_CHECK_RESP, resp); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_set_qp_credit_flag(struct ibv_qp *qp, uint64_t credit_flag) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_QP_OBJ, ++ ZXDH_IB_METHOD_QP_SET_CREDIT_FLAG, 2); ++ fill_attr_in_obj(cmd, ZXDH_IB_ATTR_QP_SET_CREDIT_FLAG_HANDLE, qp->handle); ++ fill_attr_in_uint64(cmd, ZXDH_IB_ATTR_QP_CREDIT_FLAG, credit_flag); ++ return execute_ioctl(qp->context, cmd); ++} ++ ++static int _zxdh_cfg_dev_parameter(struct ibv_context *ibctx, ++ struct zxdh_cfg_dev_parameter_req *req) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEVICE_EX, ++ ZXDH_IB_METHOD_DEV_CFG_PARAMETER, 1); ++ fill_attr_in_ptr(cmd, ZXDH_IB_ATTR_DEV_CFG_PARAMETER, req); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_show_res_map(struct ibv_context *ibctx, ++ struct zxdh_db_show_res_map_req *req, ++ struct zxdh_db_show_res_map_resp *resp) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEVICE_EX, ++ ZXDH_IB_METHOD_DEV_SHOW_RES_MAP, 2); ++ fill_attr_in_ptr(cmd, ZXDH_IB_ATTR_DEV_SHOW_RES_MAP, req); ++ fill_attr_out_ptr(cmd, ZXDH_IB_ATTR_DEV_SHOW_RES_MAP_RESP, resp); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int __zxdh_read_ram(struct ibv_context *ibctx, ++ struct zxdh_read_ram_req *req) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEVICE_EX, ++ ZXDH_IB_METHOD_DEV_READ_RAM, 1); ++ fill_attr_in_ptr(cmd, ZXDH_IB_ATTR_DEV_READ_RAM, req); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_devx_modify_cq(struct ibv_cq *cq, uint32_t overflow_check_en) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_CQ_OBJ, ++ ZXDH_IB_METHOD_CQ_MODIFY_OVERFLOW_CHECK_EN, 2); ++ fill_attr_in_obj(cmd, ZXDH_IB_ATTR_MODIFY_CQ_HANDLE, cq->handle); ++ fill_attr_in_uint32(cmd, ZXDH_IB_ATTR_MODIFY_CQ_OVERFLOW_CHECK_EN, overflow_check_en); ++ return execute_ioctl(cq->context, cmd); ++} ++ ++static int _zxdh_devx_get_peer_dev_sbdf(struct ibv_context *context, uint64_t *peer_pci_bdf) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEVICE_PRIV, ++ ZXDH_IB_METHOD_GET_PEER_DEV_SBDF, 1); ++ fill_attr_out_ptr(cmd, ZXDH_IB_ATTR_GET_PEER_DEV_SBDF, peer_pci_bdf); ++ return execute_ioctl(context, cmd); ++} ++ + static struct zxdh_uvcontext_ops zxdh_ctx_ops = { + .modify_qp_udp_sport = _zxdh_modify_qp_udp_sport, + .get_log_trace_switch = _zxdh_get_log_trace_switch, +@@ -126,6 +372,37 @@ + .query_qpc = _zxdh_query_qpc, + .modify_qpc = _zxdh_modify_qpc, + .reset_qp = _zxdh_reset_qp, ++ .cap_start = _zxdh_cap_start, ++ .cap_stop = _zxdh_cap_stop, ++ .cap_free = _zxdh_cap_free, ++ .mp_capture = _zxdh_mp_capture, ++ .mp_capture_gqp = _zxdh_mp_capture_gqp, ++ .mp_get_data = _zxdh_mp_get_data, ++ .mp_capture_clear = _zxdh_mp_capture_clear, ++ .get_active_vhca_gqps = _zxdh_get_active_vhca_gqps, ++ .rdma_hmc_query = _zxdh_rdma_hmc_query, ++ .get_obj_data = _zxdh_get_object_data, ++ .get_cc_basic_info = _zxdh_get_cc_basic_info, ++ .rdma_health_check = _zxdh_rdma_health_check, ++ .cfg_dev_parameter = _zxdh_cfg_dev_parameter, ++ .show_res_map = _zxdh_show_res_map, ++ .read_ram = __zxdh_read_ram, ++ .set_qp_credit_flag = _zxdh_set_qp_credit_flag, ++ .zxdh_devx_get_cq_init_size = _zxdh_devx_get_cq_init_size, ++ .zxdh_devx_create_cq = _zxdh_devx_create_cq, ++ .zxdh_devx_destroy_cq = _zxdh_devx_destroy_cq, ++ .zxdh_devx_destroy_qp = _zxdh_devx_destroy_qp, ++ .zxdh_devx_get_qp_init_size = _zxdh_devx_get_qp_init_size, ++ .zxdh_devx_modify_qp = _zxdh_devx_modify_qp, ++ .zxdh_devx_create_qp = _zxdh_devx_create_qp, ++ .zxdh_devx_modify_cq = _zxdh_devx_modify_cq, ++ .rdma_switch_query = _zxdh_rdma_switch_query, ++ .rdma_switch_config = _zxdh_rdma_switch_config, ++ .zxdh_devx_reg_mr = _zxdh_devx_reg_mr, ++ .zxdh_devx_dereg_mr = _zxdh_devx_dereg_mr, ++ .zxdh_devx_create_cq_with_ext_buf = _zxdh_devx_create_cq_with_ext_buf, ++ .zxdh_devx_create_qp_with_ext_buf = _zxdh_devx_create_qp_with_ext_buf, ++ .zxdh_devx_get_peer_dev_sbdf = _zxdh_devx_get_peer_dev_sbdf, + }; + + static inline struct zxdh_uvcontext *to_zxdhtx(struct ibv_context *ibctx) +@@ -195,6 +472,479 @@ + return dvops->set_log_trace_switch(context, switch_status); + } + ++int zxdh_cap_start(struct ibv_context *context, struct zxdh_cap_cfg *cap_cfg, ++ struct zxdh_rdma_cap_pa *cap_pa) ++{ ++ uint32_t shift; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ ++ if (!dvops) ++ return EOPNOTSUPP; ++ ++ shift = get_log2(cap_cfg->size); ++ if (shift == 0) { ++ return EINVAL; ++ } ++ cap_cfg->size = 1 << shift; ++ if (cap_cfg->size > ZXDH_HOST_DATA_CAP_MEM_SIZE) { ++ return EINVAL; ++ } ++ if (cap_cfg->size > ZXDH_HOST_DATA_CAP_TXRX_MEM_SIZE) { ++ iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ if (!dvops || (dev_attrs->rdma_tool_flags & ZXDH_RDMA_TOOL_IOVA_CAP) == 0) { ++ printf("rdma_tool_flags:0x%x, zxdh_cap_start with iova not supported!\n", ++ dev_attrs->rdma_tool_flags); ++ return EOPNOTSUPP; ++ } ++ } ++ ++ return dvops->cap_start(context, cap_cfg, cap_pa); ++} ++ ++int zxdh_cap_stop(struct ibv_context *context, uint8_t param) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ ++ if (!dvops) ++ return EOPNOTSUPP; ++ ++ return dvops->cap_stop(context, param); ++} ++ ++int zxdh_cap_free(struct ibv_context *context, uint8_t param) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ ++ if (!dvops) ++ return EOPNOTSUPP; ++ ++ return dvops->cap_free(context, param); ++} ++ ++int zxdh_mp_capture(struct ibv_context *context, ++ struct zxdh_mp_cap_cfg *cap_mp_cfg, ++ struct zxdh_mp_cap_gqp *cap_mp_gqp) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ ++ if (!dvops) ++ return EOPNOTSUPP; ++ ++ return dvops->mp_capture(context, cap_mp_cfg, cap_mp_gqp); ++} ++ ++int zxdh_mp_capture_gqp(struct ibv_context *context, ++ struct zxdh_mp_cap_gqp_cfg *cfg, ++ struct zxdh_mp_cap_gqp *cap_mp_gqp) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ ++ if (!dvops) ++ return EOPNOTSUPP; ++ ++ iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ if ((dev_attrs->rdma_tool_flags & ZXDH_RDMA_TOOL_PRIV_EXT) == 0) { ++ printf("rdma_tool_flags:0x%x, zxdh_mp_capture_gqp not supported!\n", ++ dev_attrs->rdma_tool_flags); ++ return EOPNOTSUPP; ++ } ++ ++ return dvops->mp_capture_gqp(context, cfg, cap_mp_gqp); ++} ++ ++int zxdh_mp_get_data(struct ibv_context *context, uint8_t param) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ ++ if (!dvops) ++ return EOPNOTSUPP; ++ ++ return dvops->mp_get_data(context, param); ++} ++ ++int zxdh_mp_capture_clear(struct ibv_context *context, ++ struct zxdh_cap_gqp *cap_gqp) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ ++ if (!dvops) ++ return EOPNOTSUPP; ++ ++ return dvops->mp_capture_clear(context, cap_gqp); ++} ++ ++int zxdh_get_active_vhca_gqps(struct ibv_context *context, ++ struct zxdh_active_vhca_gqps *resp) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ ++ if (!dvops) ++ return EOPNOTSUPP; ++ ++ return dvops->get_active_vhca_gqps(context, resp); ++} ++ ++int zxdh_rdma_hmc_query(struct ibv_context *context, ++ struct zxdh_context_req *req, ++ struct zxdh_context_resp *resp) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ if (!dvops) ++ return EOPNOTSUPP; ++ ++ return dvops->rdma_hmc_query(context, req, resp); ++} ++ ++int zxdh_rdma_switch_query(struct ibv_context *context, ++ struct query_rdma_switch_req *req, ++ struct query_rdma_switch_resp *resp) ++{ ++ struct zxdh_uvcontext_ops *dvops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ ++ if (context == NULL || req == NULL || resp == NULL) { ++ return EINVAL; ++ } ++ dvops = to_zxdhtx(context)->cxt_ops; ++ iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ if (!dvops || (dev_attrs->rdma_tool_flags & ZXDH_RDMA_SWITCH) == 0) ++ return EOPNOTSUPP; ++ ++ return dvops->rdma_switch_query(context, req, resp); ++} ++ ++int zxdh_rdma_switch_config(struct ibv_context *context, ++ struct config_rdma_switch_req *req) ++{ ++ struct zxdh_uvcontext_ops *dvops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ ++ if (context == NULL || req == NULL) { ++ return EINVAL; ++ } ++ dvops = to_zxdhtx(context)->cxt_ops; ++ iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ if (!dvops || (dev_attrs->rdma_tool_flags & ZXDH_RDMA_SWITCH) == 0) ++ return EOPNOTSUPP; ++ ++ return dvops->rdma_switch_config(context, req); ++} ++ ++int zxdh_get_object_data(struct ibv_context *context, ++ struct zxdh_get_object_data_req *req, ++ struct zxdh_get_object_data_resp *resp) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ ++ iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ if (!dvops || (dev_attrs->rdma_tool_flags & ZXDH_GET_HW_OBJECT_DATA) == 0) ++ return EOPNOTSUPP; ++ ++ return dvops->get_obj_data(context, req, resp); ++} ++ ++int zxdh_get_cc_basic_info(struct ibv_context *context, ++ struct zxdh_cc_basic_info *resp) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ ++ if (!dvops) ++ return EOPNOTSUPP; ++ ++ return dvops->get_cc_basic_info(context, resp); ++} ++ ++int zxdh_rdma_health_check(struct ibv_context *context, ++ struct zxdh_health_check_req *req, ++ struct zxdh_health_check_resp *resp) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ ++ iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ if (!dvops || (dev_attrs->rdma_tool_flags & ZXDH_CHECK_HW_HEALTH) == 0){ ++ printf("rdma_tool_flags:0x%x, rdma_health_check not supported!\n", dev_attrs->rdma_tool_flags); ++ return EOPNOTSUPP; ++ } ++ ++ return dvops->rdma_health_check(context, req, resp); ++} ++ ++int zxdh_cfg_dev_parameter(struct ibv_context *context, ++ struct zxdh_cfg_dev_parameter_req *req) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ ++ iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ if (!dvops || (dev_attrs->rdma_tool_flags & ZXDH_RDMA_TOOL_CFG_DEV_PARAM) == 0){ ++ printf("rdma_tool_flags:0x%x, zxdh_cfg_dev_parameter:%u not supported!\n", ++ dev_attrs->rdma_tool_flags, req->type); ++ return EOPNOTSUPP; ++ } ++ ++ return dvops->cfg_dev_parameter(context, req); ++} ++ ++int zxdh_show_res_map(struct ibv_context *context, ++ struct zxdh_db_show_res_map_req *req, ++ struct zxdh_db_show_res_map_resp *resp) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ ++ iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ if (!dvops || (dev_attrs->rdma_tool_flags & ZXDH_RDMA_TOOL_SHOW_RES_MAP) == 0) { ++ printf("rdma_tool_flags:0x%x, zxdh_show_res_map not supported!\n", ++ dev_attrs->rdma_tool_flags); ++ return EOPNOTSUPP; ++ } ++ ++ return dvops->show_res_map(context, req, resp); ++} ++ ++int zxdh_read_ram(struct ibv_context *context, struct zxdh_read_ram_req *req) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ ++ iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ if (!dvops || ++ (dev_attrs->rdma_tool_flags & ZXDH_RDMA_TOOL_READ_RAM) == 0) { ++ printf("rdma_tool_flags:0x%x, zxdh_read_ram not supported!\n", ++ dev_attrs->rdma_tool_flags); ++ return EOPNOTSUPP; ++ } ++ return dvops->read_ram(context, req); ++} ++ ++int zxdh_set_qp_credit_flag(struct ibv_context *context, ++ struct ibv_qp *qp, uint64_t credit_flag) ++{ ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(qp->context)->cxt_ops; ++ iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ ++ if (!dvops || !dvops->set_qp_credit_flag || ++ (iwvctx->dev_attrs.rdma_tool_flags & ZXDH_RDMA_SET_CREDIT_CAP) == 0) { ++ return EOPNOTSUPP; ++ } ++ return dvops->set_qp_credit_flag(qp, credit_flag); ++} ++ ++int zxdh_devx_get_cq_init_size(struct ibv_context *context, ++ struct zxdh_devx_cq_init_attr *attr, ++ struct zxdh_devx_init_cq_resp *resp) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ ++ if (!dvops || !dvops->zxdh_devx_get_cq_init_size) ++ return EOPNOTSUPP; ++ ++ return dvops->zxdh_devx_get_cq_init_size(context, attr, resp); ++} ++ ++struct zxdh_devx_cq *zxdh_devx_create_cq(struct ibv_context *context, int cqe_num, ++ struct ibv_comp_channel *channel, int comp_vector, ++ struct zxdh_devx_cq_buffers *buffer) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ ++ if (!dvops || !dvops->zxdh_devx_create_cq) ++ return NULL; ++ ++ return dvops->zxdh_devx_create_cq(context, cqe_num, channel, ++ comp_vector, buffer); ++} ++ ++int zxdh_devx_destroy_cq(struct zxdh_devx_cq *cq) ++{ ++ if (!cq->verbs_cq.cq.context) ++ return EOPNOTSUPP; ++ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(cq->verbs_cq.cq.context)->cxt_ops; ++ ++ if (!dvops || !dvops->zxdh_devx_destroy_cq) ++ return EOPNOTSUPP; ++ ++ return dvops->zxdh_devx_destroy_cq(cq); ++} ++ ++int zxdh_devx_modify_qp(struct zxdh_devx_qp *qp, struct ibv_qp_attr *attr, ++ int attr_mask) ++{ ++ struct zxdh_uvcontext_ops *dvops = ++ to_zxdhtx(qp->verbs_qp.qp.context)->cxt_ops; ++ ++ if (!dvops || !dvops->zxdh_devx_modify_qp) ++ return -EOPNOTSUPP; ++ ++ return dvops->zxdh_devx_modify_qp(qp, attr, attr_mask); ++} ++ ++int zxdh_devx_destroy_qp(struct zxdh_devx_qp *qp) ++{ ++ struct zxdh_uvcontext_ops *dvops = ++ to_zxdhtx(qp->verbs_qp.qp.context)->cxt_ops; ++ ++ if (!dvops || !dvops->zxdh_devx_destroy_qp) ++ return -EOPNOTSUPP; ++ ++ return dvops->zxdh_devx_destroy_qp(qp); ++} ++ ++struct zxdh_devx_qp *zxdh_devx_create_qp(struct ibv_pd *pd, ++ struct ibv_qp_init_attr *qp_init_attr, ++ struct zxdh_devx_qp_buffers *buffer) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(pd->context)->cxt_ops; ++ ++ if (!dvops || !dvops->zxdh_devx_create_qp) { ++ errno = -EOPNOTSUPP; ++ return NULL; ++ } ++ ++ return dvops->zxdh_devx_create_qp(pd, qp_init_attr, buffer); ++} ++ ++int zxdh_devx_get_qp_init_size(struct ibv_context *context, ++ struct zxdh_devx_qp_init_attr *attr, ++ struct zxdh_devx_init_qp_resp *resp) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ ++ if (!dvops || !dvops->zxdh_devx_get_qp_init_size) ++ return -EOPNOTSUPP; ++ ++ return dvops->zxdh_devx_get_qp_init_size(context, attr, resp); ++} ++ ++int zxdh_devx_modify_cq(struct ibv_cq *cq, uint32_t overflow_check_en) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(cq->context)->cxt_ops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ iwvctx = container_of(cq->context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ if (!dvops || (dev_attrs->rdma_tool_flags & ZXDH_RDMA_TOOL_DEVX_MODIFY_CQ) == 0) { ++ printf("rdma_tool_flags:0x%x, zxdh_devx_modify_cq not supported!\n", dev_attrs->rdma_tool_flags); ++ return EOPNOTSUPP; ++ } ++ return dvops->zxdh_devx_modify_cq(cq, overflow_check_en); ++} ++ ++struct ibv_mr *zxdh_devx_reg_mr(struct ibv_pd *pd, void *host_addr, size_t length, uint64_t dev_addr, unsigned int access) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(pd->context)->cxt_ops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ iwvctx = container_of(pd->context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ if (!dvops || !dvops->zxdh_devx_reg_mr || ++ (dev_attrs->rdma_tool_flags & ZXDH_RDMA_TOOL_DEVX_EXT_MEM) == 0) { ++ printf("rdma_tool_flags:0x%x, zxdh_devx_reg_mr not supported!\n", dev_attrs->rdma_tool_flags); ++ return NULL; ++ } ++ return dvops->zxdh_devx_reg_mr(pd, host_addr, length, dev_addr, access); ++} ++ ++int zxdh_devx_dereg_mr(struct ibv_mr *mr) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(mr->context)->cxt_ops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ iwvctx = container_of(mr->context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ ++ if (!dvops || !dvops->zxdh_devx_dereg_mr || ++ (dev_attrs->rdma_tool_flags & ZXDH_RDMA_TOOL_DEVX_EXT_MEM) == 0) { ++ printf("rdma_tool_flags:0x%x, zxdh_devx_dereg_mr not supported!\n", dev_attrs->rdma_tool_flags); ++ return -EOPNOTSUPP; ++ } ++ return dvops->zxdh_devx_dereg_mr(mr); ++} ++ ++struct ibv_cq *zxdh_devx_create_cq_with_ext_buf(struct ibv_context *context, ++ int cqe_num, ++ struct ibv_comp_channel *channel, ++ int comp_vector, ++ struct zxdh_ext_buf *buf) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ if (!dvops || !dvops->zxdh_devx_create_cq_with_ext_buf|| ++ (dev_attrs->rdma_tool_flags & ZXDH_RDMA_TOOL_DEVX_EXT_MEM) == 0){ ++ printf("rdma_tool_flags:0x%x, zxdh_devx_create_cq_with_ext_buf not supported!\n", dev_attrs->rdma_tool_flags); ++ return NULL; ++ } ++ ++ return dvops->zxdh_devx_create_cq_with_ext_buf(context, cqe_num, channel, comp_vector, buf); ++} ++ ++struct ibv_qp *zxdh_devx_create_qp_with_ext_buf(struct ibv_pd *pd, ++ struct ibv_qp_init_attr *attr, ++ struct zxdh_ext_buf *buf) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(pd->context)->cxt_ops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ ++ iwvctx = container_of(pd->context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ ++ if (!dvops || !dvops->zxdh_devx_create_qp_with_ext_buf || ++ (dev_attrs->rdma_tool_flags & ZXDH_RDMA_TOOL_DEVX_EXT_MEM) == 0) { ++ printf("rdma_tool_flags:0x%x, create qp with ext buf not supported\n", ++ dev_attrs->rdma_tool_flags); ++ return NULL; ++ } ++ ++ return dvops->zxdh_devx_create_qp_with_ext_buf(pd, attr, buf); ++} ++ ++int zxdh_devx_get_peer_dev_sbdf(struct ibv_context *ibv_ctx, uint64_t *peer_pci_bdf) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(ibv_ctx)->cxt_ops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ ++ iwvctx = container_of(ibv_ctx, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ ++ if (peer_pci_bdf == NULL || dvops == NULL || ++ dvops->zxdh_devx_get_peer_dev_sbdf == NULL || ++ (dev_attrs->rdma_tool_flags & ZXDH_RDMA_TOOL_DEVX_EXT_MEM) == 0) { ++ printf("rdma_tool_flags:0x%x, get peer dev sbdf not supported!\n", dev_attrs->rdma_tool_flags); ++ return -EOPNOTSUPP; ++ } ++ return dvops->zxdh_devx_get_peer_dev_sbdf(ibv_ctx, peer_pci_bdf); ++} ++ + void add_private_ops(struct zxdh_uvcontext *iwvctx) + { + iwvctx->cxt_ops = &zxdh_ctx_ops; +diff -ruN baseline/providers/zrdma/private_verbs_cmd.c.tmp b/providers/zrdma/private_verbs_cmd.c.tmp +--- a/providers/zrdma/private_verbs_cmd.c.tmp 1970-01-01 08:00:00.000000000 +0800 ++++ b/providers/zrdma/private_verbs_cmd.c.tmp 2026-08-06 15:29:37.847716922 +0800 +@@ -0,0 +1,951 @@ ++// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB ++/* Copyright (c) 2024 ZTE Corporation. All rights reserved. */ ++#include ++#include ++#include "private_verbs_cmd.h" ++#include "zxdh_devx.h" ++ ++static inline uint32_t get_log2(uint32_t x) ++{ ++ uint32_t r = 0; ++ for (x >>= 1; x > 0; x >>= 1) ++ r++; ++ return r; ++} ++ ++static void copy_query_qpc(struct zxdh_query_qpc_resp *resp, ++ struct zxdh_rdma_qpc *qpc) ++{ ++ qpc->ack_err_flag = resp->ack_err_flag; ++ qpc->retry_flag = resp->retry_flag; ++ qpc->rnr_retry_flag = resp->rnr_retry_flag; ++ qpc->cur_retry_count = resp->cur_retry_count; ++ qpc->retry_cqe_sq_opcode = resp->retry_cqe_sq_opcode; ++ qpc->err_flag = resp->err_flag; ++ qpc->package_err_flag = resp->package_err_flag; ++ qpc->recv_err_flag = resp->recv_err_flag; ++ qpc->tx_last_ack_psn = resp->tx_last_ack_psn; ++ qpc->retry_count = resp->retry_count; ++ qpc->read_retry_flag = resp->read_retry_flag; ++} ++ ++static int _zxdh_query_qpc(struct ibv_qp *qp, struct zxdh_rdma_qpc *qpc) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_QP_OBJ, ++ ZXDH_IB_METHOD_QP_QUERY_QPC, 2); ++ int ret; ++ struct zxdh_query_qpc_resp resp_ex = { 0 }; ++ ++ fill_attr_in_obj(cmd, ZXDH_IB_ATTR_QP_QUERY_HANDLE, qp->handle); ++ fill_attr_out_ptr(cmd, ZXDH_IB_ATTR_QP_QUERY_RESP, &resp_ex); ++ ++ ret = execute_ioctl(qp->context, cmd); ++ if (ret) ++ return ret; ++ ++ copy_query_qpc(&resp_ex, qpc); ++ return 0; ++} ++ ++static void copy_modify_qpc_fields(struct zxdh_modify_qpc_req *req_cmd, ++ uint64_t attr_mask, ++ struct zxdh_rdma_qpc *qpc) ++{ ++ if (attr_mask & ZXDH_TX_READ_RETRY_FLAG_SET) { ++ req_cmd->retry_flag = qpc->retry_flag; ++ req_cmd->rnr_retry_flag = qpc->rnr_retry_flag; ++ req_cmd->read_retry_flag = qpc->read_retry_flag; ++ req_cmd->cur_retry_count = qpc->cur_retry_count; ++ } ++ if (attr_mask & ZXDH_RETRY_CQE_SQ_OPCODE) ++ req_cmd->retry_cqe_sq_opcode = qpc->retry_cqe_sq_opcode; ++ ++ if (attr_mask & ZXDH_ERR_FLAG_SET) { ++ req_cmd->err_flag = qpc->err_flag; ++ req_cmd->ack_err_flag = qpc->ack_err_flag; ++ } ++ if (attr_mask & ZXDH_PACKAGE_ERR_FLAG) ++ req_cmd->package_err_flag = qpc->package_err_flag; ++} ++ ++static int _zxdh_reset_qp(struct ibv_qp *qp, uint64_t opcode) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_QP_OBJ, ++ ZXDH_IB_METHOD_QP_RESET_QP, 2); ++ fill_attr_in_obj(cmd, ZXDH_IB_ATTR_QP_RESET_QP_HANDLE, qp->handle); ++ fill_attr_in_uint64(cmd, ZXDH_IB_ATTR_QP_RESET_OP_CODE, opcode); ++ return execute_ioctl(qp->context, cmd); ++} ++ ++static int _zxdh_modify_qpc(struct ibv_qp *qp, struct zxdh_rdma_qpc *qpc, ++ uint64_t qpc_mask) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_QP_OBJ, ++ ZXDH_IB_METHOD_QP_MODIFY_QPC, 3); ++ struct zxdh_modify_qpc_req req = { 0 }; ++ ++ copy_modify_qpc_fields(&req, qpc_mask, qpc); ++ fill_attr_in_obj(cmd, ZXDH_IB_ATTR_QP_QUERY_HANDLE, qp->handle); ++ fill_attr_in_uint64(cmd, ZXDH_IB_ATTR_QP_MODIFY_QPC_MASK, qpc_mask); ++ fill_attr_in_ptr(cmd, ZXDH_IB_ATTR_QP_MODIFY_QPC_REQ, &req); ++ return execute_ioctl(qp->context, cmd); ++} ++ ++static int _zxdh_modify_qp_udp_sport(struct ibv_context *ibctx, ++ uint16_t udp_sport, uint32_t qpn) ++{ ++ if (udp_sport <= MIN_UDP_SPORT || qpn <= MIN_QP_QPN) ++ return -EINVAL; ++ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_QP_OBJ, ++ ZXDH_IB_METHOD_QP_MODIFY_UDP_SPORT, 2); ++ fill_attr_in(cmd, ZXDH_IB_ATTR_QP_UDP_PORT, &udp_sport, ++ sizeof(udp_sport)); ++ fill_attr_in_uint32(cmd, ZXDH_IB_ATTR_QP_QPN, qpn); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_get_log_trace_switch(struct ibv_context *ibctx, ++ uint8_t *switch_status) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEV, ++ ZXDH_IB_METHOD_DEV_GET_LOG_TRACE, 1); ++ ++ fill_attr_out_ptr(cmd, ZXDH_IB_ATTR_DEV_GET_LOG_TARCE_SWITCH, ++ switch_status); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_set_log_trace_switch(struct ibv_context *ibctx, ++ uint8_t switch_status) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEV, ++ ZXDH_IB_METHOD_DEV_SET_LOG_TRACE, 1); ++ fill_attr_in(cmd, ZXDH_IB_ATTR_DEV_SET_LOG_TARCE_SWITCH, &switch_status, ++ sizeof(switch_status)); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_cap_start(struct ibv_context *ibctx, ++ struct zxdh_cap_cfg *cap_cfg, ++ struct zxdh_rdma_cap_pa *cap_pa) ++{ ++ int ret; ++ struct zxdh_cap_start_resp resp = { 0 }; ++ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEV, ++ ZXDH_IB_METHOD_DEV_CAP_START, 2); ++ fill_attr_in_ptr(cmd, ZXDH_IB_ATTR_DEV_CAP_START, cap_cfg); ++ fill_attr_out_ptr(cmd, ZXDH_IB_ATTR_DEV_CAP_START_RESP, &resp); ++ ret = execute_ioctl(ibctx, cmd); ++ if (ret == 0) { ++ cap_pa->cap_pa_node0 = resp.cap_pa_node0; ++ cap_pa->cap_pa_node1 = resp.cap_pa_node1; ++ } else { ++ printf("execute_ioctl failed,%d\n", ret); ++ } ++ ++ return ret; ++} ++ ++static int _zxdh_cap_stop(struct ibv_context *ibctx, uint8_t param) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEV, ++ ZXDH_IB_METHOD_DEV_CAP_STOP, 1); ++ fill_attr_in(cmd, ZXDH_IB_ATTR_DEV_CAP_STOP, ¶m, sizeof(param)); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_cap_free(struct ibv_context *ibctx, uint8_t param) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEV, ++ ZXDH_IB_METHOD_DEV_CAP_FREE, 1); ++ fill_attr_in(cmd, ZXDH_IB_ATTR_DEV_CAP_FREE, ¶m, sizeof(param)); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_mp_capture(struct ibv_context *ibctx, ++ struct zxdh_mp_cap_cfg *cap_mp_cfg, ++ struct zxdh_mp_cap_gqp *cap_mp_gqp) ++{ ++ struct zxdh_mp_cap_resp resp = { 0 }; ++ int ret; ++ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEV, ++ ZXDH_IB_METHOD_DEV_MP_CAP, 2); ++ ++ fill_attr_in_ptr(cmd, ZXDH_IB_ATTR_DEV_MP_CAP, cap_mp_cfg); ++ fill_attr_out_ptr(cmd, ZXDH_IB_ATTR_DEV_MP_CAP_RESP, &resp); ++ ret = execute_ioctl(ibctx, cmd); ++ ++ if (ret == 0) { ++ cap_mp_gqp->mcode_type = resp.mcode_type; ++ cap_mp_gqp->cap_pa = resp.cap_pa; ++ cap_mp_gqp->gqp_num = resp.cap_gqp_num; ++ memcpy(cap_mp_gqp->gqpid, resp.cap_gqpid, ++ sizeof(cap_mp_gqp->gqpid)); ++ } else { ++ printf("execute_ioctl for mp capture failed,%d\n", ret); ++ } ++ ++ return ret; ++} ++ ++static int _zxdh_mp_capture_gqp(struct ibv_context *ibctx, ++ struct zxdh_mp_cap_gqp_cfg *cfg, ++ struct zxdh_mp_cap_gqp *cap_mp_gqp) ++{ ++ struct zxdh_mp_cap_resp resp = { 0 }; ++ int ret; ++ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEVICE_PRIV, ++ ZXDH_IB_METHOD_DEV_MP_CAP_GQP, 2); ++ ++ fill_attr_in_ptr(cmd, ZXDH_IB_ATTR_DEV_MP_CAP_GQP, cfg); ++ fill_attr_out_ptr(cmd, ZXDH_IB_ATTR_DEV_MP_CAP_GQP_RESP, &resp); ++ ret = execute_ioctl(ibctx, cmd); ++ if (ret) { ++ printf("execute_ioctl for mp capture by gqp failed,%d\n", ret); ++ return ret; ++ } ++ ++ cap_mp_gqp->mcode_type = resp.mcode_type; ++ cap_mp_gqp->cap_pa = resp.cap_pa; ++ cap_mp_gqp->gqp_num = resp.cap_gqp_num; ++ memcpy(cap_mp_gqp->gqpid, resp.cap_gqpid, ++ sizeof(cap_mp_gqp->gqpid)); ++ ++ return 0; ++} ++ ++static int _zxdh_mp_get_data(struct ibv_context *ibctx, uint8_t param) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEV, ++ ZXDH_IB_METHOD_DEV_MP_GET_DATA, 1); ++ ++ fill_attr_in(cmd, ZXDH_IB_ATTR_DEV_MP_GET_DATA, ¶m, sizeof(param)); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_mp_capture_clear(struct ibv_context *ibctx, ++ struct zxdh_cap_gqp *cap_gqp) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEV, ++ ZXDH_IB_METHOD_DEV_MP_CAP_CLEAR, 1); ++ ++ fill_attr_in_ptr(cmd, ZXDH_IB_ATTR_DEV_MP_CAP_CLEAR, cap_gqp); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_get_active_vhca_gqps(struct ibv_context *ibctx, ++ struct zxdh_active_vhca_gqps *resp) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEVICE_EX, ++ ZXDH_IB_METHOD_DEV_GET_ACT_VHCA_GQPS, 1); ++ ++ fill_attr_out_ptr(cmd, ZXDH_IB_ATTR_DEV_GET_ACT_VHCA_GQPS_RESP, resp); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_rdma_hmc_query(struct ibv_context *ibctx, ++ struct zxdh_context_req *req, ++ struct zxdh_context_resp *resp) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEVICE_EX, ++ ZXDH_IB_METHOD_DEV_GET_HMC, 2); ++ fill_attr_in_ptr(cmd, ZXDH_IB_ATTR_DEV_GET_HMC, req); ++ fill_attr_out_ptr(cmd, ZXDH_IB_ATTR_DEV_GET_HMC_RESP, resp); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_rdma_switch_query(struct ibv_context *ibctx, ++ struct query_rdma_switch_req *req, ++ struct query_rdma_switch_resp *resp) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEVICE_OBJ, ++ ZXDH_IB_METHOD_DEV_QUERY_RDMA_SWITCH, 2); ++ fill_attr_in_ptr(cmd, ZXDH_IB_ATTR_DEV_GET_SWITCH, req); ++ fill_attr_out_ptr(cmd, ZXDH_IB_ATTR_DEV_GET_SWITCH_RESP, resp); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_rdma_switch_config(struct ibv_context *ibctx, ++ struct config_rdma_switch_req *req) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEVICE_OBJ, ++ ZXDH_IB_METHOD_DEV_CONFIG_RDMA_SWITCH, 1); ++ fill_attr_in_ptr(cmd, ZXDH_IB_ATTR_DEV_CONFIG_SWITCH, req); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_get_object_data(struct ibv_context *ibctx, ++ struct zxdh_get_object_data_req *req, ++ struct zxdh_get_object_data_resp *resp) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEVICE_EX, ++ ZXDH_IB_METHOD_DEV_GET_OBJ_DATA, 2); ++ fill_attr_in_ptr(cmd, ZXDH_IB_ATTR_DEV_GET_OBJ_DATA, req); ++ fill_attr_out_ptr(cmd, ZXDH_IB_ATTR_DEV_GET_OBJ_DATA_RESP, resp); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_get_cc_basic_info(struct ibv_context *ibctx, ++ struct zxdh_cc_basic_info *resp) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEVICE_EX, ++ ZXDH_IB_METHOD_DEV_GET_CC_BASIC_INFO, 1); ++ ++ fill_attr_out_ptr(cmd, ZXDH_IB_ATTR_DEV_GET_CC_BASIC_INFO_RESP, resp); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_rdma_health_check(struct ibv_context *ibctx, ++ struct zxdh_health_check_req *req, ++ struct zxdh_health_check_resp *resp) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEVICE_EX, ++ ZXDH_IB_METHOD_DEV_HEALTH_CHECK, 2); ++ fill_attr_in_ptr(cmd, ZXDH_IB_ATTR_DEV_HEALTH_CHECK, req); ++ fill_attr_out_ptr(cmd, ZXDH_IB_ATTR_DEV_HEALTH_CHECK_RESP, resp); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_set_qp_credit_flag(struct ibv_qp *qp, uint64_t credit_flag) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_QP_OBJ, ++ ZXDH_IB_METHOD_QP_SET_CREDIT_FLAG, 2); ++ fill_attr_in_obj(cmd, ZXDH_IB_ATTR_QP_SET_CREDIT_FLAG_HANDLE, qp->handle); ++ fill_attr_in_uint64(cmd, ZXDH_IB_ATTR_QP_CREDIT_FLAG, credit_flag); ++ return execute_ioctl(qp->context, cmd); ++} ++ ++static int _zxdh_cfg_dev_parameter(struct ibv_context *ibctx, ++ struct zxdh_cfg_dev_parameter_req *req) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEVICE_EX, ++ ZXDH_IB_METHOD_DEV_CFG_PARAMETER, 1); ++ fill_attr_in_ptr(cmd, ZXDH_IB_ATTR_DEV_CFG_PARAMETER, req); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_show_res_map(struct ibv_context *ibctx, ++ struct zxdh_db_show_res_map_req *req, ++ struct zxdh_db_show_res_map_resp *resp) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEVICE_EX, ++ ZXDH_IB_METHOD_DEV_SHOW_RES_MAP, 2); ++ fill_attr_in_ptr(cmd, ZXDH_IB_ATTR_DEV_SHOW_RES_MAP, req); ++ fill_attr_out_ptr(cmd, ZXDH_IB_ATTR_DEV_SHOW_RES_MAP_RESP, resp); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int __zxdh_read_ram(struct ibv_context *ibctx, ++ struct zxdh_read_ram_req *req) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEVICE_EX, ++ ZXDH_IB_METHOD_DEV_READ_RAM, 1); ++ fill_attr_in_ptr(cmd, ZXDH_IB_ATTR_DEV_READ_RAM, req); ++ return execute_ioctl(ibctx, cmd); ++} ++ ++static int _zxdh_devx_modify_cq(struct ibv_cq *cq, uint32_t overflow_check_en) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_CQ_OBJ, ++ ZXDH_IB_METHOD_CQ_MODIFY_OVERFLOW_CHECK_EN, 2); ++ fill_attr_in_obj(cmd, ZXDH_IB_ATTR_MODIFY_CQ_HANDLE, cq->handle); ++ fill_attr_in_uint32(cmd, ZXDH_IB_ATTR_MODIFY_CQ_OVERFLOW_CHECK_EN, overflow_check_en); ++ return execute_ioctl(cq->context, cmd); ++} ++ ++static int _zxdh_devx_get_peer_dev_sbdf(struct ibv_context *context, uint64_t *peer_pci_bdf) ++{ ++ DECLARE_COMMAND_BUFFER(cmd, ZXDH_IB_OBJECT_DEVICE_PRIV, ++ ZXDH_IB_METHOD_GET_PEER_DEV_SBDF, 1); ++ fill_attr_out_ptr(cmd, ZXDH_IB_ATTR_GET_PEER_DEV_SBDF, peer_pci_bdf); ++ return execute_ioctl(context, cmd); ++} ++ ++static struct zxdh_uvcontext_ops zxdh_ctx_ops = { ++ .modify_qp_udp_sport = _zxdh_modify_qp_udp_sport, ++ .get_log_trace_switch = _zxdh_get_log_trace_switch, ++ .set_log_trace_switch = _zxdh_set_log_trace_switch, ++ .query_qpc = _zxdh_query_qpc, ++ .modify_qpc = _zxdh_modify_qpc, ++ .reset_qp = _zxdh_reset_qp, ++ .cap_start = _zxdh_cap_start, ++ .cap_stop = _zxdh_cap_stop, ++ .cap_free = _zxdh_cap_free, ++ .mp_capture = _zxdh_mp_capture, ++ .mp_capture_gqp = _zxdh_mp_capture_gqp, ++ .mp_get_data = _zxdh_mp_get_data, ++ .mp_capture_clear = _zxdh_mp_capture_clear, ++ .get_active_vhca_gqps = _zxdh_get_active_vhca_gqps, ++ .rdma_hmc_query = _zxdh_rdma_hmc_query, ++ .get_obj_data = _zxdh_get_object_data, ++ .get_cc_basic_info = _zxdh_get_cc_basic_info, ++ .rdma_health_check = _zxdh_rdma_health_check, ++ .cfg_dev_parameter = _zxdh_cfg_dev_parameter, ++ .show_res_map = _zxdh_show_res_map, ++ .read_ram = __zxdh_read_ram, ++ .set_qp_credit_flag = _zxdh_set_qp_credit_flag, ++ .zxdh_devx_get_cq_init_size = _zxdh_devx_get_cq_init_size, ++ .zxdh_devx_create_cq = _zxdh_devx_create_cq, ++ .zxdh_devx_destroy_cq = _zxdh_devx_destroy_cq, ++ .zxdh_devx_destroy_qp = _zxdh_devx_destroy_qp, ++ .zxdh_devx_get_qp_init_size = _zxdh_devx_get_qp_init_size, ++ .zxdh_devx_modify_qp = _zxdh_devx_modify_qp, ++ .zxdh_devx_create_qp = _zxdh_devx_create_qp, ++ .zxdh_devx_modify_cq = _zxdh_devx_modify_cq, ++ .rdma_switch_query = _zxdh_rdma_switch_query, ++ .rdma_switch_config = _zxdh_rdma_switch_config, ++ .zxdh_devx_reg_mr = _zxdh_devx_reg_mr, ++ .zxdh_devx_dereg_mr = _zxdh_devx_dereg_mr, ++ .zxdh_devx_create_cq_with_ext_buf = _zxdh_devx_create_cq_with_ext_buf, ++ .zxdh_devx_create_qp_with_ext_buf = _zxdh_devx_create_qp_with_ext_buf, ++ .zxdh_devx_get_peer_dev_sbdf = _zxdh_devx_get_peer_dev_sbdf, ++}; ++ ++static inline struct zxdh_uvcontext *to_zxdhtx(struct ibv_context *ibctx) ++{ ++ return container_of(ibctx, struct zxdh_uvcontext, ibv_ctx.context); ++} ++ ++int zxdh_reset_qp(struct ibv_qp *qp, uint64_t opcode) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(qp->context)->cxt_ops; ++ ++ if (!dvops || !dvops->reset_qp) ++ return -EOPNOTSUPP; ++ return dvops->reset_qp(qp, opcode); ++} ++ ++int zxdh_modify_qpc(struct ibv_qp *qp, struct zxdh_rdma_qpc *qpc, ++ uint64_t qpc_mask) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(qp->context)->cxt_ops; ++ ++ if (!dvops || !dvops->modify_qpc) ++ return -EOPNOTSUPP; ++ return dvops->modify_qpc(qp, qpc, qpc_mask); ++} ++ ++int zxdh_query_qpc(struct ibv_qp *qp, struct zxdh_rdma_qpc *qpc) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(qp->context)->cxt_ops; ++ ++ if (!dvops || !dvops->query_qpc) ++ return -EOPNOTSUPP; ++ ++ return dvops->query_qpc(qp, qpc); ++} ++ ++int zxdh_modify_qp_udp_sport(struct ibv_context *context, uint16_t udp_sport, ++ uint32_t qpn) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ ++ if (!dvops || !dvops->modify_qp_udp_sport) ++ return -EOPNOTSUPP; ++ ++ return dvops->modify_qp_udp_sport(context, udp_sport, qpn); ++} ++ ++int zxdh_get_log_trace_switch(struct ibv_context *context, ++ enum switch_status *switch_status) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ ++ if (!dvops || !dvops->get_log_trace_switch) ++ return -EOPNOTSUPP; ++ ++ return dvops->get_log_trace_switch(context, (uint8_t *)switch_status); ++} ++ ++int zxdh_set_log_trace_switch(struct ibv_context *context, ++ enum switch_status switch_status) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ ++ if (!dvops || !dvops->set_log_trace_switch) ++ return -EOPNOTSUPP; ++ ++ return dvops->set_log_trace_switch(context, switch_status); ++} ++ ++int zxdh_cap_start(struct ibv_context *context, struct zxdh_cap_cfg *cap_cfg, ++ struct zxdh_rdma_cap_pa *cap_pa) ++{ ++ uint32_t shift; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ ++ if (!dvops) ++ return EOPNOTSUPP; ++ ++ shift = get_log2(cap_cfg->size); ++ if (shift == 0) { ++ return EINVAL; ++ } ++ cap_cfg->size = 1 << shift; ++ if (cap_cfg->size > ZXDH_HOST_DATA_CAP_MEM_SIZE) { ++ return EINVAL; ++ } ++ if (cap_cfg->size > ZXDH_HOST_DATA_CAP_TXRX_MEM_SIZE) { ++ iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ if (!dvops || (dev_attrs->rdma_tool_flags & ZXDH_RDMA_TOOL_IOVA_CAP) == 0) { ++ printf("rdma_tool_flags:0x%x, zxdh_cap_start with iova not supported!\n", ++ dev_attrs->rdma_tool_flags); ++ return EOPNOTSUPP; ++ } ++ } ++ ++ return dvops->cap_start(context, cap_cfg, cap_pa); ++} ++ ++int zxdh_cap_stop(struct ibv_context *context, uint8_t param) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ ++ if (!dvops) ++ return EOPNOTSUPP; ++ ++ return dvops->cap_stop(context, param); ++} ++ ++int zxdh_cap_free(struct ibv_context *context, uint8_t param) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ ++ if (!dvops) ++ return EOPNOTSUPP; ++ ++ return dvops->cap_free(context, param); ++} ++ ++int zxdh_mp_capture(struct ibv_context *context, ++ struct zxdh_mp_cap_cfg *cap_mp_cfg, ++ struct zxdh_mp_cap_gqp *cap_mp_gqp) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ ++ if (!dvops) ++ return EOPNOTSUPP; ++ ++ return dvops->mp_capture(context, cap_mp_cfg, cap_mp_gqp); ++} ++ ++int zxdh_mp_capture_gqp(struct ibv_context *context, ++ struct zxdh_mp_cap_gqp_cfg *cfg, ++ struct zxdh_mp_cap_gqp *cap_mp_gqp) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ ++ if (!dvops) ++ return EOPNOTSUPP; ++ ++ iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ if ((dev_attrs->rdma_tool_flags & ZXDH_RDMA_TOOL_PRIV_EXT) == 0) { ++ printf("rdma_tool_flags:0x%x, zxdh_mp_capture_gqp not supported!\n", ++ dev_attrs->rdma_tool_flags); ++ return EOPNOTSUPP; ++ } ++ ++ return dvops->mp_capture_gqp(context, cfg, cap_mp_gqp); ++} ++ ++int zxdh_mp_get_data(struct ibv_context *context, uint8_t param) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ ++ if (!dvops) ++ return EOPNOTSUPP; ++ ++ return dvops->mp_get_data(context, param); ++} ++ ++int zxdh_mp_capture_clear(struct ibv_context *context, ++ struct zxdh_cap_gqp *cap_gqp) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ ++ if (!dvops) ++ return EOPNOTSUPP; ++ ++ return dvops->mp_capture_clear(context, cap_gqp); ++} ++ ++int zxdh_get_active_vhca_gqps(struct ibv_context *context, ++ struct zxdh_active_vhca_gqps *resp) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ ++ if (!dvops) ++ return EOPNOTSUPP; ++ ++ return dvops->get_active_vhca_gqps(context, resp); ++} ++ ++int zxdh_rdma_hmc_query(struct ibv_context *context, ++ struct zxdh_context_req *req, ++ struct zxdh_context_resp *resp) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ if (!dvops) ++ return EOPNOTSUPP; ++ ++ return dvops->rdma_hmc_query(context, req, resp); ++} ++ ++int zxdh_rdma_switch_query(struct ibv_context *context, ++ struct query_rdma_switch_req *req, ++ struct query_rdma_switch_resp *resp) ++{ ++ struct zxdh_uvcontext_ops *dvops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ ++ if (context == NULL || req == NULL || resp == NULL) { ++ return EINVAL; ++ } ++ dvops = to_zxdhtx(context)->cxt_ops; ++ iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ if (!dvops || (dev_attrs->rdma_tool_flags & ZXDH_RDMA_SWITCH) == 0) ++ return EOPNOTSUPP; ++ ++ return dvops->rdma_switch_query(context, req, resp); ++} ++ ++int zxdh_rdma_switch_config(struct ibv_context *context, ++ struct config_rdma_switch_req *req) ++{ ++ struct zxdh_uvcontext_ops *dvops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ ++ if (context == NULL || req == NULL) { ++ return EINVAL; ++ } ++ dvops = to_zxdhtx(context)->cxt_ops; ++ iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ if (!dvops || (dev_attrs->rdma_tool_flags & ZXDH_RDMA_SWITCH) == 0) ++ return EOPNOTSUPP; ++ ++ return dvops->rdma_switch_config(context, req); ++} ++ ++int zxdh_get_object_data(struct ibv_context *context, ++ struct zxdh_get_object_data_req *req, ++ struct zxdh_get_object_data_resp *resp) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ ++ iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ if (!dvops || (dev_attrs->rdma_tool_flags & ZXDH_GET_HW_OBJECT_DATA) == 0) ++ return EOPNOTSUPP; ++ ++ return dvops->get_obj_data(context, req, resp); ++} ++ ++int zxdh_get_cc_basic_info(struct ibv_context *context, ++ struct zxdh_cc_basic_info *resp) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ ++ if (!dvops) ++ return EOPNOTSUPP; ++ ++ return dvops->get_cc_basic_info(context, resp); ++} ++ ++int zxdh_rdma_health_check(struct ibv_context *context, ++ struct zxdh_health_check_req *req, ++ struct zxdh_health_check_resp *resp) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ ++ iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ if (!dvops || (dev_attrs->rdma_tool_flags & ZXDH_CHECK_HW_HEALTH) == 0){ ++ printf("rdma_tool_flags:0x%x, rdma_health_check not supported!\n", dev_attrs->rdma_tool_flags); ++ return EOPNOTSUPP; ++ } ++ ++ return dvops->rdma_health_check(context, req, resp); ++} ++ ++int zxdh_cfg_dev_parameter(struct ibv_context *context, ++ struct zxdh_cfg_dev_parameter_req *req) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ ++ iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ if (!dvops || (dev_attrs->rdma_tool_flags & ZXDH_RDMA_TOOL_CFG_DEV_PARAM) == 0){ ++ printf("rdma_tool_flags:0x%x, zxdh_cfg_dev_parameter:%u not supported!\n", ++ dev_attrs->rdma_tool_flags, req->type); ++ return EOPNOTSUPP; ++ } ++ ++ return dvops->cfg_dev_parameter(context, req); ++} ++ ++int zxdh_show_res_map(struct ibv_context *context, ++ struct zxdh_db_show_res_map_req *req, ++ struct zxdh_db_show_res_map_resp *resp) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ ++ iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ if (!dvops || (dev_attrs->rdma_tool_flags & ZXDH_RDMA_TOOL_SHOW_RES_MAP) == 0) { ++ printf("rdma_tool_flags:0x%x, zxdh_show_res_map not supported!\n", ++ dev_attrs->rdma_tool_flags); ++ return EOPNOTSUPP; ++ } ++ ++ return dvops->show_res_map(context, req, resp); ++} ++ ++int zxdh_read_ram(struct ibv_context *context, struct zxdh_read_ram_req *req) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ ++ iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ if (!dvops || ++ (dev_attrs->rdma_tool_flags & ZXDH_RDMA_TOOL_READ_RAM) == 0) { ++ printf("rdma_tool_flags:0x%x, zxdh_read_ram not supported!\n", ++ dev_attrs->rdma_tool_flags); ++ return EOPNOTSUPP; ++ } ++ return dvops->read_ram(context, req); ++} ++ ++int zxdh_set_qp_credit_flag(struct ibv_context *context, ++ struct ibv_qp *qp, uint64_t credit_flag) ++{ ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(qp->context)->cxt_ops; ++ iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ ++ if (!dvops || !dvops->set_qp_credit_flag || ++ (iwvctx->dev_attrs.rdma_tool_flags & ZXDH_RDMA_SET_CREDIT_CAP) == 0) { ++ return EOPNOTSUPP; ++ } ++ return dvops->set_qp_credit_flag(qp, credit_flag); ++} ++ ++int zxdh_devx_get_cq_init_size(struct ibv_context *context, ++ struct zxdh_devx_cq_init_attr *attr, ++ struct zxdh_devx_init_cq_resp *resp) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ ++ if (!dvops || !dvops->zxdh_devx_get_cq_init_size) ++ return EOPNOTSUPP; ++ ++ return dvops->zxdh_devx_get_cq_init_size(context, attr, resp); ++} ++ ++struct zxdh_devx_cq *zxdh_devx_create_cq(struct ibv_context *context, int cqe_num, ++ struct ibv_comp_channel *channel, int comp_vector, ++ struct zxdh_devx_cq_buffers *buffer) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ ++ if (!dvops || !dvops->zxdh_devx_create_cq) ++ return NULL; ++ ++ return dvops->zxdh_devx_create_cq(context, cqe_num, channel, ++ comp_vector, buffer); ++} ++ ++int zxdh_devx_destroy_cq(struct zxdh_devx_cq *cq) ++{ ++ if (!cq->verbs_cq.cq.context) ++ return EOPNOTSUPP; ++ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(cq->verbs_cq.cq.context)->cxt_ops; ++ ++ if (!dvops || !dvops->zxdh_devx_destroy_cq) ++ return EOPNOTSUPP; ++ ++ return dvops->zxdh_devx_destroy_cq(cq); ++} ++ ++int zxdh_devx_modify_qp(struct zxdh_devx_qp *qp, struct ibv_qp_attr *attr, ++ int attr_mask) ++{ ++ struct zxdh_uvcontext_ops *dvops = ++ to_zxdhtx(qp->verbs_qp.qp.context)->cxt_ops; ++ ++ if (!dvops || !dvops->zxdh_devx_modify_qp) ++ return -EOPNOTSUPP; ++ ++ return dvops->zxdh_devx_modify_qp(qp, attr, attr_mask); ++} ++ ++int zxdh_devx_destroy_qp(struct zxdh_devx_qp *qp) ++{ ++ struct zxdh_uvcontext_ops *dvops = ++ to_zxdhtx(qp->verbs_qp.qp.context)->cxt_ops; ++ ++ if (!dvops || !dvops->zxdh_devx_destroy_qp) ++ return -EOPNOTSUPP; ++ ++ return dvops->zxdh_devx_destroy_qp(qp); ++} ++ ++struct zxdh_devx_qp *zxdh_devx_create_qp(struct ibv_pd *pd, ++ struct ibv_qp_init_attr *qp_init_attr, ++ struct zxdh_devx_qp_buffers *buffer) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(pd->context)->cxt_ops; ++ ++ if (!dvops || !dvops->zxdh_devx_create_qp) { ++ errno = -EOPNOTSUPP; ++ return NULL; ++ } ++ ++ return dvops->zxdh_devx_create_qp(pd, qp_init_attr, buffer); ++} ++ ++int zxdh_devx_get_qp_init_size(struct ibv_context *context, ++ struct zxdh_devx_qp_init_attr *attr, ++ struct zxdh_devx_init_qp_resp *resp) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ ++ if (!dvops || !dvops->zxdh_devx_get_qp_init_size) ++ return -EOPNOTSUPP; ++ ++ return dvops->zxdh_devx_get_qp_init_size(context, attr, resp); ++} ++ ++int zxdh_devx_modify_cq(struct ibv_cq *cq, uint32_t overflow_check_en) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(cq->context)->cxt_ops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ iwvctx = container_of(cq->context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ if (!dvops || (dev_attrs->rdma_tool_flags & ZXDH_RDMA_TOOL_DEVX_MODIFY_CQ) == 0) { ++ printf("rdma_tool_flags:0x%x, zxdh_devx_modify_cq not supported!\n", dev_attrs->rdma_tool_flags); ++ return EOPNOTSUPP; ++ } ++ return dvops->zxdh_devx_modify_cq(cq, overflow_check_en); ++} ++ ++struct ibv_mr *zxdh_devx_reg_mr(struct ibv_pd *pd, void *host_addr, size_t length, uint64_t dev_addr, unsigned int access) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(pd->context)->cxt_ops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ iwvctx = container_of(pd->context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ if (!dvops || !dvops->zxdh_devx_reg_mr || ++ (dev_attrs->rdma_tool_flags & ZXDH_RDMA_TOOL_DEVX_EXT_MEM) == 0) { ++ printf("rdma_tool_flags:0x%x, zxdh_devx_reg_mr not supported!\n", dev_attrs->rdma_tool_flags); ++ return NULL; ++ } ++ return dvops->zxdh_devx_reg_mr(pd, host_addr, length, dev_addr, access); ++} ++ ++int zxdh_devx_dereg_mr(struct ibv_mr *mr) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(mr->context)->cxt_ops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ iwvctx = container_of(mr->context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ ++ if (!dvops || !dvops->zxdh_devx_dereg_mr || ++ (dev_attrs->rdma_tool_flags & ZXDH_RDMA_TOOL_DEVX_EXT_MEM) == 0) { ++ printf("rdma_tool_flags:0x%x, zxdh_devx_dereg_mr not supported!\n", dev_attrs->rdma_tool_flags); ++ return -EOPNOTSUPP; ++ } ++ return dvops->zxdh_devx_dereg_mr(mr); ++} ++ ++struct ibv_cq *zxdh_devx_create_cq_with_ext_buf(struct ibv_context *context, ++ int cqe_num, ++ struct ibv_comp_channel *channel, ++ int comp_vector, ++ struct zxdh_ext_buf *buf) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(context)->cxt_ops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ if (!dvops || !dvops->zxdh_devx_create_cq_with_ext_buf|| ++ (dev_attrs->rdma_tool_flags & ZXDH_RDMA_TOOL_DEVX_EXT_MEM) == 0){ ++ printf("rdma_tool_flags:0x%x, zxdh_devx_create_cq_with_ext_buf not supported!\n", dev_attrs->rdma_tool_flags); ++ return NULL; ++ } ++ ++ return dvops->zxdh_devx_create_cq_with_ext_buf(context, cqe_num, channel, comp_vector, buf); ++} ++ ++struct ibv_qp *zxdh_devx_create_qp_with_ext_buf(struct ibv_pd *pd, ++ struct ibv_qp_init_attr *attr, ++ struct zxdh_ext_buf *buf) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(pd->context)->cxt_ops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ ++ iwvctx = container_of(pd->context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ ++ if (!dvops || !dvops->zxdh_devx_create_qp_with_ext_buf || ++ (dev_attrs->rdma_tool_flags & ZXDH_RDMA_TOOL_DEVX_EXT_MEM) == 0) { ++ printf("rdma_tool_flags:0x%x, create qp with ext buf not supported\n", ++ dev_attrs->rdma_tool_flags); ++ return NULL; ++ } ++ ++ return dvops->zxdh_devx_create_qp_with_ext_buf(pd, attr, buf); ++} ++ ++int zxdh_devx_get_peer_dev_sbdf(struct ibv_context *ibv_ctx, uint64_t *peer_pci_bdf) ++{ ++ struct zxdh_uvcontext_ops *dvops = to_zxdhtx(ibv_ctx)->cxt_ops; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ ++ iwvctx = container_of(ibv_ctx, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ ++ if (peer_pci_bdf == NULL || dvops == NULL || ++ dvops->zxdh_devx_get_peer_dev_sbdf == NULL || ++ (dev_attrs->rdma_tool_flags & ZXDH_RDMA_TOOL_DEVX_EXT_MEM) == 0) { ++ printf("rdma_tool_flags:0x%x, get peer dev sbdf not supported!\n", dev_attrs->rdma_tool_flags); ++ return -EOPNOTSUPP; ++ } ++ return dvops->zxdh_devx_get_peer_dev_sbdf(ibv_ctx, peer_pci_bdf); ++} ++ ++void add_private_ops(struct zxdh_uvcontext *iwvctx) ++{ ++ iwvctx->cxt_ops = &zxdh_ctx_ops; ++} +diff -ruN baseline/providers/zrdma/private_verbs_cmd.h b/providers/zrdma/private_verbs_cmd.h +--- a/providers/zrdma/private_verbs_cmd.h 2026-08-06 15:37:55.338861801 +0800 ++++ b/providers/zrdma/private_verbs_cmd.h 2026-08-06 15:29:37.610700645 +0800 +@@ -3,9 +3,26 @@ + #ifndef ZXDH_RDMA_PRIVATE_VERBS_CMD_H + #define ZXDH_RDMA_PRIVATE_VERBS_CMD_H + +-#include "main.h" ++#include "zxdh_zrdma.h" + #include "zxdh_dv.h" + ++enum zxdh_rdma_tool_flags { ++ ZXDH_QP_EXTEND_OP = 1 << 0, ++ ZXDH_CAPTURE = 1 << 1, ++ ZXDH_GET_HW_DATA = 1 << 2, ++ ZXDH_GET_HW_OBJECT_DATA = 1 << 3, ++ ZXDH_CHECK_HW_HEALTH = 1 << 4, ++ ZXDH_RDMA_TOOL_CFG_DEV_PARAM = 1 << 5, ++ ZXDH_RDMA_TOOL_SHOW_RES_MAP = 1 << 5, ++ ZXDH_RDMA_TOOL_READ_RAM = 1 << 6, ++ ZXDH_RDMA_TOOL_DEVX_MODIFY_CQ = 1 << 7, ++ ZXDH_RDMA_SET_CREDIT_CAP = 1 << 8, ++ ZXDH_RDMA_SWITCH = 1 << 9, ++ ZXDH_RDMA_TOOL_DEVX_EXT_MEM = 1 << 10, ++ ZXDH_RDMA_TOOL_PRIV_EXT = 1 << 11, ++ ZXDH_RDMA_TOOL_IOVA_CAP = 1 << 12, ++}; ++ + struct zxdh_uvcontext_ops { + int (*modify_qp_udp_sport)(struct ibv_context *ibctx, + uint16_t udp_sport, uint32_t qpn); +@@ -17,6 +34,76 @@ + int (*modify_qpc)(struct ibv_qp *qp, struct zxdh_rdma_qpc *qpc, + uint64_t qpc_mask); + int (*reset_qp)(struct ibv_qp *qp, uint64_t opcode); ++ int (*cap_start)(struct ibv_context *ibctx, ++ struct zxdh_cap_cfg *cap_cfg, ++ struct zxdh_rdma_cap_pa *cap_pa); ++ int (*cap_stop)(struct ibv_context *ibctx, uint8_t param); ++ int (*cap_free)(struct ibv_context *ibctx, uint8_t param); ++ int (*mp_capture)(struct ibv_context *ibctx, ++ struct zxdh_mp_cap_cfg *cap_mp_cfg, ++ struct zxdh_mp_cap_gqp *cap_mp_gqp); ++ int (*mp_capture_gqp)(struct ibv_context *ibctx, ++ struct zxdh_mp_cap_gqp_cfg *cfg, ++ struct zxdh_mp_cap_gqp *cap_mp_gqp); ++ int (*mp_get_data)(struct ibv_context *ibctx, uint8_t param); ++ int (*mp_capture_clear)(struct ibv_context *context, ++ struct zxdh_cap_gqp *cap_gqp); ++ int (*get_active_vhca_gqps)(struct ibv_context *context, ++ struct zxdh_active_vhca_gqps *resp); ++ int (*rdma_hmc_query)(struct ibv_context *context, ++ struct zxdh_context_req *req, ++ struct zxdh_context_resp *resp); ++ int (*get_obj_data)(struct ibv_context *context, ++ struct zxdh_get_object_data_req *req, ++ struct zxdh_get_object_data_resp *resp); ++ int (*get_cc_basic_info)(struct ibv_context *context, ++ struct zxdh_cc_basic_info *resp); ++ int (*rdma_health_check)(struct ibv_context *context, ++ struct zxdh_health_check_req *req, ++ struct zxdh_health_check_resp *resp); ++ int (*cfg_dev_parameter)(struct ibv_context *context, ++ struct zxdh_cfg_dev_parameter_req *req); ++ int (*show_res_map)(struct ibv_context *context, ++ struct zxdh_db_show_res_map_req *req, ++ struct zxdh_db_show_res_map_resp *resp); ++ int (*read_ram)(struct ibv_context *context, ++ struct zxdh_read_ram_req *req); ++ int (*set_qp_credit_flag)(struct ibv_qp *qp, ++ uint64_t credit_flag); ++ int (*zxdh_devx_get_cq_init_size)(struct ibv_context *context, ++ struct zxdh_devx_cq_init_attr *attr, ++ struct zxdh_devx_init_cq_resp *resp); ++ struct zxdh_devx_cq *(*zxdh_devx_create_cq)( ++ struct ibv_context *context, int cqe_num, ++ struct ibv_comp_channel *channel, int comp_vector, ++ struct zxdh_devx_cq_buffers *buffer); ++ int (*zxdh_devx_destroy_cq)(struct zxdh_devx_cq *cq); ++ int (*zxdh_devx_get_qp_init_size)(struct ibv_context *context, ++ struct zxdh_devx_qp_init_attr *attr, ++ struct zxdh_devx_init_qp_resp *resp); ++ struct zxdh_devx_qp *(*zxdh_devx_create_qp)( ++ struct ibv_pd *pd, struct ibv_qp_init_attr *attr, ++ struct zxdh_devx_qp_buffers *buffer); ++ int (*zxdh_devx_destroy_qp)(struct zxdh_devx_qp *devx_qp); ++ int (*zxdh_devx_modify_qp)(struct zxdh_devx_qp *devx_qp, ++ struct ibv_qp_attr *attr, int attr_mask); ++ int (*zxdh_devx_modify_cq)(struct ibv_cq *cq, uint32_t overflow_check_en); ++ int (*rdma_switch_query)(struct ibv_context *context, ++ struct query_rdma_switch_req *req, ++ struct query_rdma_switch_resp *resp); ++ int (*rdma_switch_config)(struct ibv_context *context, ++ struct config_rdma_switch_req *req); ++ struct ibv_mr *(*zxdh_devx_reg_mr)(struct ibv_pd *pd, void *host_addr, size_t length, uint64_t dev_addr, unsigned int access); ++ int (*zxdh_devx_dereg_mr)(struct ibv_mr *mr); ++ struct ibv_cq *(*zxdh_devx_create_cq_with_ext_buf)(struct ibv_context *context, ++ int cqe_num, ++ struct ibv_comp_channel *channel, ++ int comp_vector, ++ struct zxdh_ext_buf *buf); ++ struct ibv_qp *(*zxdh_devx_create_qp_with_ext_buf)(struct ibv_pd *pd, ++ struct ibv_qp_init_attr *attr, ++ struct zxdh_ext_buf *buf); ++ int (*zxdh_devx_get_peer_dev_sbdf)(struct ibv_context *ibv_ctx, uint64_t *peer_pci_bdf); + }; + + void add_private_ops(struct zxdh_uvcontext *iwvctx); +diff -ruN baseline/providers/zrdma/wqe_rate_limit.c b/providers/zrdma/wqe_rate_limit.c +--- a/providers/zrdma/wqe_rate_limit.c 1970-01-01 08:00:00.000000000 +0800 ++++ b/providers/zrdma/wqe_rate_limit.c 2026-08-06 15:29:36.663635605 +0800 +@@ -0,0 +1,453 @@ ++// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB ++/* Copyright (c) 2024 ZTE Corporation. All rights reserved. */ ++ ++#include ++#include ++#include ++ ++#include "zxdh_zrdma.h" ++#include "zxdh_abi.h" ++#include "wqe_rate_limit.h" ++#if defined(__x86_64__) || defined (__i386__) ++ ++static int cpu_mhz = 0; ++static int first_qp_flag = 0; ++static unsigned int wqe_total_size = 0; ++static unsigned int max_cnt = 0; ++static cycles_t gap_cycles = 0; ++static const int power_of_two_table[] = {1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576, 2097152, 4194304, 8388608}; ++wqe_rate_limit_info_t g_wqe_rate_limit_info = {0}; ++ ++static wqe_rate_limit_info_t *init_wqe_rate_limit_info(void) ++{ ++ wqe_rate_limit_info_t *temp = &g_wqe_rate_limit_info; ++ wqe_rate_limit_info_t *info = (wqe_rate_limit_info_t *)malloc(sizeof(wqe_rate_limit_info_t)); ++ if (info == NULL) { ++ printf("%s malloc fail!\n", __FUNCTION__); ++ return NULL; ++ } ++ info->gap_deadline = 0; ++ info->vhca_wqe_db = NULL; ++ info->vhca_qp_num = 0; ++ info->vhca_rate = 0; ++ info->total_wqe_size = 0; ++ info->iwuqp_qpn = 0; ++ info->param1 = 0; ++ info->param2 = 0; ++ info->wqe_rate_limit = 0; ++ info->first_wqe_flag = 1; ++ info->current_log2_size = 0; ++ info->next = NULL; ++ ++ while (temp->next != NULL) { ++ temp = temp->next; ++ } ++ temp->next = info; ++ return info; ++} ++ ++static wqe_rate_limit_info_t *find_wqe_rate_limit_info_by_vhca_wqe_db(unsigned int *vhca_wqe_db) ++{ ++ wqe_rate_limit_info_t *temp = &g_wqe_rate_limit_info; ++ while (temp != NULL) { ++ if (temp->vhca_wqe_db == vhca_wqe_db) { ++ return temp; ++ } ++ ++ temp = temp->next; ++ } ++ return NULL; ++} ++ ++static void delete_wqe_rate_limit_info(wqe_rate_limit_info_t *info) ++{ ++ wqe_rate_limit_info_t *temp = &g_wqe_rate_limit_info; ++ while (temp->next != NULL) { ++ if (temp->next == info) { ++ if (info->next != NULL) { ++ temp->next = info->next; ++ } else { ++ temp->next = NULL; ++ } ++ ++ free(info); ++ return; ++ } ++ ++ temp = temp->next; ++ } ++ return; ++} ++ ++ ++ ++static inline cycles_t get_cycles(void) ++{ ++ unsigned low = 0, high = 0; ++ unsigned long long val = 0; ++ asm volatile ("rdtsc" : "=a" (low), "=d" (high)); ++ val = high; ++ val = (val << 32) | low; ++ return val; ++} ++ ++static double sample_get_cpu_mhz(void) ++{ ++ struct timeval tv1, tv2; ++ cycles_t start; ++ double sx = 0, sy = 0, sxx = 0, syy = 0, sxy = 0; ++ double tx, ty; ++ int i; ++ ++ /* Regression: y = a + b x */ ++ long x[MEASUREMENTS]; ++ cycles_t y[MEASUREMENTS]; ++ double a; /* system call overhead in cycles */ ++ double b; /* cycles per microsecond */ ++ double r_2; ++ ++ for (i = 0; i < MEASUREMENTS; ++i) { ++ start = get_cycles(); ++ ++ if (gettimeofday(&tv1, NULL)) { ++ fprintf(stderr, "gettimeofday failed.\n"); ++ return 0; ++ } ++ ++ do { ++ if (gettimeofday(&tv2, NULL)) { ++ fprintf(stderr, "gettimeofday failed.\n"); ++ return 0; ++ } ++ } while ((tv2.tv_sec - tv1.tv_sec) * 1000000 + ++ (tv2.tv_usec - tv1.tv_usec) < USECSTART + i * USECSTEP); ++ ++ x[i] = (tv2.tv_sec - tv1.tv_sec) * 1000000 + ++ tv2.tv_usec - tv1.tv_usec; ++ y[i] = get_cycles() - start; ++ if (DEBUG_DATA) ++ fprintf(stderr, "x=%ld y=%lld\n", x[i], (long long)y[i]); ++ } ++ ++ for (i = 0; i < MEASUREMENTS; ++i) { ++ tx = x[i]; ++ ty = y[i]; ++ sx += tx; ++ sy += ty; ++ sxx += tx * tx; ++ syy += ty * ty; ++ sxy += tx * ty; ++ } ++ ++ b = (MEASUREMENTS * sxy - sx * sy) / (MEASUREMENTS * sxx - sx * sx); ++ a = (sy - b * sx) / MEASUREMENTS; ++ ++ if (DEBUG) ++ fprintf(stderr, "a = %g\n", a); ++ if (DEBUG) ++ fprintf(stderr, "b = %g\n", b); ++ if (DEBUG) ++ fprintf(stderr, "a / b = %g\n", a / b); ++ r_2 = (MEASUREMENTS * sxy - sx * sy) * (MEASUREMENTS * sxy - sx * sy) / ++ (MEASUREMENTS * sxx - sx * sx) / ++ (MEASUREMENTS * syy - sy * sy); ++ ++ if (DEBUG) ++ fprintf(stderr, "r^2 = %g\n", r_2); ++ if (r_2 < 0.9) { ++ fprintf(stderr, "Correlation coefficient r^2: %g < 0.9\n", r_2); ++ return 0; ++ } ++ ++ return b; ++} ++ ++ ++#if !defined(__s390x__) && !defined(__s390__) ++static double proc_get_cpu_mhz(int no_cpu_freq_warn) ++{ ++ FILE* f; ++ char buf[256]; ++ double mhz = 0.0; ++ int print_flag = 0; ++ double delta; ++ ++ #if defined(__FreeBSD__) ++ f = popen("/sbin/sysctl hw.clockrate","r"); ++ #else ++ f = fopen("/proc/cpuinfo","r"); ++ #endif ++ ++ if (!f) return 0.0; ++ ++ while(fgets(buf, sizeof(buf), f)) { ++ double m; ++ int rc; ++ ++ #if defined (__ia64__) ++ /* Use the ITC frequency on IA64 */ ++ rc = sscanf(buf, "itc MHz : %lf", &m); ++ #elif defined (__PPC__) || defined (__PPC64__) ++ /* PPC has a different format as well */ ++ rc = sscanf(buf, "clock : %lf", &m); ++ #else ++ #if defined (__FreeBSD__) ++ rc = sscanf(buf, "hw.clockrate: %lf", &m); ++ #else ++ rc = sscanf(buf, "cpu MHz : %lf", &m); ++ #endif ++ #endif ++ ++ if (rc != 1) ++ continue; ++ ++ if (mhz < 0.000001) { ++ mhz = m; ++ continue; ++ } ++ delta = mhz > m ? mhz - m : m - mhz; ++ if ((delta / mhz > 0.02) && (print_flag ==0)) { ++ print_flag = 1; ++ continue; ++ } ++ } ++ ++#if defined(__FreeBSD__) ++ pclose(f); ++#else ++ fclose(f); ++#endif ++ return mhz; ++} ++#endif ++ ++static double get_cpu_mhz(int no_cpu_freq_warn) ++{ ++ #if defined(__s390x__) || defined(__s390__) ++ return sample_get_cpu_mhz(); ++ #else ++ double sample, proc, delta; ++ sample = sample_get_cpu_mhz(); ++ proc = proc_get_cpu_mhz(no_cpu_freq_warn); ++ #ifdef __aarch64__ ++ if (proc < 1) ++ proc = sample; ++ #endif ++ if (proc < 0.000001 || sample < 0.000001) return 0; ++ ++ ++ delta = proc > sample ? proc - sample : sample - proc; ++ if (delta / proc > 0.02) return sample; ++ ++ ++ return proc; ++#endif ++} ++ ++ ++int get_ib_device_rate(const char *ibdev_path, int port, int *rate) ++{ ++ char path[MAX_PATH]; ++ FILE *file; ++ char rate_str[64]; ++ char *token = NULL; ++ int ret = 0; ++ ++ ret = snprintf(path, MAX_PATH, "%s/ports/%d/rate", ibdev_path, port); ++ if (ret < 0) { ++ return ret; ++ } ++ ++ file = fopen(path, "r"); ++ if (!file) { ++ perror("Failed to open rate file"); ++ return -1; ++ } ++ ++ if (fgets(rate_str, sizeof(rate_str), file) == NULL) { ++ perror("Failed to read rate"); ++ fclose(file); ++ return -1; ++ } ++ ++ fclose(file); ++ ++ // rate_str = "200 Gb/sec (4X HDR)" ++ token = strtok(rate_str, " "); ++ if (token) { ++ *rate = atoi(token); ++ } else { ++ perror("Failed to get rate"); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++static int is_power_of_two_table(unsigned int n) ++{ ++ for (unsigned int i = 0; i < sizeof(power_of_two_table) / sizeof(power_of_two_table[0]); i++) ++ { ++ if (power_of_two_table[i] == n) ++ { ++ return i; ++ } ++ } ++ ++ return -1; ++} ++ ++enum rate_limit_flags rdma_rate_limit_flag(struct wqe_rate_limit_info *wqe_rate_limit_info, unsigned int size) ++{ ++ if (size <= 0) ++ { ++ return RATE_LIMIT_ON; ++ } ++ ++ if ((size & (size - 1)) == 0) ++ { ++ unsigned int log2_size = 0; ++ unsigned int temp = size; ++ ++ log2_size = is_power_of_two_table(temp); ++ ++ if (log2_size == -1) ++ return RATE_LIMIT_OFF; ++ ++ if(wqe_rate_limit_info->first_wqe_flag) ++ { ++ if (log2_size != 1) ++ return RATE_LIMIT_OFF; ++ wqe_rate_limit_info->current_log2_size = 1; ++ wqe_rate_limit_info->first_wqe_flag = 0; ++ } ++ else ++ { ++ if (log2_size == wqe_rate_limit_info->current_log2_size) ++ { ++ return RATE_LIMIT_ON; ++ } ++ else if (log2_size == wqe_rate_limit_info->current_log2_size + 1) ++ { ++ wqe_rate_limit_info->current_log2_size = log2_size; ++ return RATE_LIMIT_ON; ++ } ++ else ++ { ++ wqe_rate_limit_info->first_wqe_flag = 1; ++ return RATE_LIMIT_OFF; ++ } ++ } ++ } ++ else ++ { ++ return RATE_LIMIT_OFF; ++ } ++ return RATE_LIMIT_ON; ++} ++ ++void rdma_wqe_rate_limit_qp_init(struct zxdh_qp_init_info *info, char *ibdev_path, struct zxdh_uqp *iwuqp, uint8_t flag) ++{ ++ wqe_rate_limit_info_t *wqe_rate_limit_info = NULL; ++ if(first_qp_flag == 0) ++ { ++ first_qp_flag = 1; ++ cpu_mhz = get_cpu_mhz(1); ++ } ++ ++ ++ wqe_rate_limit_info = find_wqe_rate_limit_info_by_vhca_wqe_db(info->wqe_alloc_db); ++ if (wqe_rate_limit_info == NULL) { ++ wqe_rate_limit_info = init_wqe_rate_limit_info(); ++ if (wqe_rate_limit_info == NULL) return; ++ ++ wqe_rate_limit_info->vhca_wqe_db = info->wqe_alloc_db; ++ if(get_ib_device_rate(ibdev_path, 1, &wqe_rate_limit_info->vhca_rate)) { ++ wqe_rate_limit_info->param1 = 0.0; ++ wqe_rate_limit_info->param2 = 0.0; ++ } else { ++ wqe_rate_limit_info->param1 = OPTIM_PARAM_MTU1024_QPN8190 * cpu_mhz * 8 / (wqe_rate_limit_info->vhca_rate * 1000); ++ wqe_rate_limit_info->param2 = OPTIM_PARAM_DEFAULT * cpu_mhz * 8 / (wqe_rate_limit_info->vhca_rate * 1000); ++ } ++ } ++ ++ wqe_rate_limit_info->wqe_rate_limit = flag; ++ wqe_rate_limit_info->vhca_qp_num++; ++ wqe_rate_limit_info->first_wqe_flag = 1; ++ wqe_rate_limit_info->current_log2_size = 0; ++ ++ iwuqp->wqe_rate_limit_info = wqe_rate_limit_info; ++} ++ ++void rdma_wqe_rate_limit_qp_exit(struct zxdh_uqp *iwuqp) ++{ ++ if(iwuqp->wqe_rate_limit_info != NULL) ++ { ++ iwuqp->wqe_rate_limit_info->vhca_qp_num--; ++ if(iwuqp->wqe_rate_limit_info->vhca_qp_num == 0) { ++ delete_wqe_rate_limit_info(iwuqp->wqe_rate_limit_info); ++ } ++ iwuqp->wqe_rate_limit_info = NULL; ++ } ++} ++ ++void rdma_qp_delay_wqe(struct zxdh_post_sq_info *info, struct zxdh_uqp *iwuqp) ++{ ++ wqe_rate_limit_info_t *wqe_rate_limit_info = iwuqp->wqe_rate_limit_info; ++ wqe_total_size = 0; ++ ++ if (wqe_rate_limit_info == NULL) return; ++ ++ ++ ++ if(wqe_rate_limit_info->iwuqp_qpn == 0) ++ { ++ wqe_rate_limit_info->iwuqp_qpn = iwuqp->qp.qp_id; ++ wqe_rate_limit_info->gap_deadline = get_cycles(); ++ } ++ ++ for (uint32_t i = 0; i < info->op.rdma_read.num_lo_sges; i++) ++ { ++ wqe_total_size += info->op.rdma_read.lo_sg_list[i].len; ++ } ++ ++ if (rdma_rate_limit_flag(wqe_rate_limit_info, wqe_total_size) == RATE_LIMIT_OFF) ++ return; ++ ++ if(wqe_rate_limit_info->iwuqp_qpn != iwuqp->qp.qp_id) ++ { ++ ++ // calculate gap_cycles ++ if((iwuqp->qp.mtu == mtu_enum_to_int(IBV_MTU_1024)) && ++ (wqe_rate_limit_info->vhca_qp_num == MTU_1024_QPN_WQE_RATE_LIMIT_MAX)) ++ { ++ // gap_cycles = 1.075 * cpu_mhz * 8 * total_wqe_size[iwuqp->rsvd] / (vhca_rate[iwuqp->rsvd] * 1000); ++ gap_cycles = wqe_rate_limit_info->param1 * wqe_rate_limit_info->total_wqe_size; ++ } ++ else ++ { ++ // gap_cycles = 1.018 * cpu_mhz * 8 * total_wqe_size[iwuqp->rsvd] / (vhca_rate[iwuqp->rsvd] * 1000); ++ gap_cycles = wqe_rate_limit_info->param2 * wqe_rate_limit_info->total_wqe_size; ++ } ++ ++ // calculate gap_deadline ++ wqe_rate_limit_info->gap_deadline += gap_cycles; ++ ++ // wait till arrive gap_deadline or MAX_TIMEOUT_CNT ++ max_cnt = 0; ++ while((wqe_rate_limit_info->gap_deadline > get_cycles()) && (max_cnt < MAX_TIMEOUT_CNT)) ++ { ++ max_cnt++; ++ } ++ ++ wqe_rate_limit_info->total_wqe_size = wqe_total_size; ++ wqe_rate_limit_info->iwuqp_qpn = iwuqp->qp.qp_id; ++ wqe_rate_limit_info->gap_deadline = get_cycles(); ++ } ++ else ++ { ++ wqe_rate_limit_info->total_wqe_size += wqe_total_size; ++ } ++} ++ ++#endif +diff -ruN baseline/providers/zrdma/wqe_rate_limit.h b/providers/zrdma/wqe_rate_limit.h +--- a/providers/zrdma/wqe_rate_limit.h 1970-01-01 08:00:00.000000000 +0800 ++++ b/providers/zrdma/wqe_rate_limit.h 2026-08-06 15:29:37.959724614 +0800 +@@ -0,0 +1,38 @@ ++/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */ ++/* Copyright (c) 2024 ZTE Corporation. All rights reserved. */ ++#ifndef WQE_RATE_LIMIT_H ++#define WQE_RATE_LIMIT_H ++ ++typedef unsigned long long cycles_t; ++ ++#define MAX_TIMEOUT_CNT 1000000000 ++ ++#define MTU_1024_QPN_WQE_RATE_LIMIT_MAX 8190 ++#define START_WQE_RATE_LIMIT_QP_TH 1900 ++#define MAX_VHCA_NUM 8 ++#define MAX_PATH 1024 ++#define OPTIM_PARAM_DEFAULT 1.018 ++#define OPTIM_PARAM_MTU1024_QPN8190 1.075 ++ ++ ++#ifndef DEBUG ++#define DEBUG 0 ++#endif ++#ifndef DEBUG_DATA ++#define DEBUG_DATA 0 ++#endif ++#define MEASUREMENTS 200 ++#define USECSTEP 10 ++#define USECSTART 100 ++ ++enum rate_limit_flags { ++ RATE_LIMIT_OFF, ++ RATE_LIMIT_ON, ++}; ++ ++void rdma_wqe_rate_limit_qp_init(struct zxdh_qp_init_info *info, char *ibdev_path, struct zxdh_uqp *iwuqp, uint8_t flag); ++void rdma_wqe_rate_limit_qp_exit(struct zxdh_uqp *iwuqp); ++void rdma_qp_delay_wqe(struct zxdh_post_sq_info *info, struct zxdh_uqp *iwuqp); ++int get_ib_device_rate(const char *ibdev_path, int port, int *rate); ++enum rate_limit_flags rdma_rate_limit_flag(struct wqe_rate_limit_info *wqe_rate_limit_info, unsigned int size); // 区分-s和-a测试场景的判断函数 ++#endif // WQE_RATE_LIMIT_H +\ 文件尾没有换行符 +diff -ruN baseline/providers/zrdma/zxdh_defs.h b/providers/zrdma/zxdh_defs.h +--- a/providers/zrdma/zxdh_defs.h 2026-08-06 15:37:55.339861870 +0800 ++++ b/providers/zrdma/zxdh_defs.h 2026-08-06 15:29:36.859649066 +0800 +@@ -7,6 +7,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -22,6 +23,10 @@ + + #define ZXDH_HW_PAGE_SIZE 4096 + #define ZXDH_HW_PAGE_SHIFT 12 ++#define ZXDH_HW_64K_PAGE_SIZE 65536 ++#define ZXDH_SYS_PAGE_SIZE (sysconf(_SC_PAGESIZE)) ++#define ZXDH_HW_PAGE_SIZE_L2D 262144 ++ + #define ZXDH_CQE_QTYPE_RQ 0 + #define ZXDH_CQE_QTYPE_SQ 1 + +@@ -40,8 +45,9 @@ + #define ZXDH_QP_FRAG_BYTESIZE 16 + #define ZXDH_SQ_WQE_BYTESIZE 32 + #define ZXDH_SRQ_WQE_MIN_SIZE 16 ++#define ZXDH_CQ_CQE_SIZE 64 + +-#define ZXDH_SQ_RSVD 258 ++#define ZXDH_SQ_RSVD 1 + #define ZXDH_RQ_RSVD 1 + #define ZXDH_SRQ_RSVD 1 + +@@ -174,6 +180,19 @@ + #define ZXDHQPSQ_MW_LEVLE2_FIRST_PBLE_INDEX GENMASK_ULL(41, 32) + #define ZXDHQPSQ_MW_LEVLE2_ROOT_PBLE_INDEX GENMASK_ULL(50, 42) + ++#define ZXDHQPSQ_FAST_REG_MR_STAG_INDEX GENMASK_ULL(31, 8) ++#define ZXDHQPSQ_FAST_REG_MR_STAG_KEY GENMASK_ULL(7, 0) ++#define ZXDHQPSQ_FAST_REG_MR_STAG_LENGTH GENMASK_ULL(45, 0) ++#define ZXDHQPSQ_FAST_REG_MR_FIRST_PM_PBL_IDX_LOW GENMASK_ULL(61, 46) ++#define ZXDHQPSQ_FAST_REG_MR_FIRST_PM_PBL_IDX_HI GENMASK_ULL(63, 51) ++#define ZXDHQPSQ_FAST_REG_PBL_ADDR GENMASK_ULL(51, 0) ++#define ZXDHQPSQ_FAST_REG_LEAF_PBL_SIZE GENMASK_ULL(41, 40) ++#define ZXDHQPSQ_FAST_REG_HOST_PAGE_SIZE GENMASK_ULL(46, 43) ++#define ZXDHQPSQ_FAST_REG_STAG_RIGHTS GENMASK_ULL(51, 47) ++#define ZXDHQPSQ_FAST_REG_VABASEDTO BIT_ULL(53) ++ ++ ++ + #define ZXDHQPSQ_BASEVA_TO_FBO ZXDH_CQPHC_QPCTX + + #define ZXDHQPSQ_LOCSTAG GENMASK_ULL(31, 0) +@@ -225,6 +244,7 @@ + (_ring).head = 0; \ + (_ring).tail = 0; \ + (_ring).size = (_size); \ ++ (_ring).used_quanta_cnt = 0; \ + } + #define ZXDH_RING_SIZE(_ring) ((_ring).size) + #define ZXDH_RING_CURRENT_HEAD(_ring) ((_ring).head) +@@ -237,6 +257,7 @@ + if (!ZXDH_RING_FULL_ERR(_ring)) { \ + (_ring).head = ((_ring).head + 1) % size; \ + (_retcode) = 0; \ ++ (_ring).used_quanta_cnt += 1; \ + } else { \ + (_retcode) = ZXDH_ERR_RING_FULL; \ + } \ +@@ -245,52 +266,35 @@ + { \ + register __u32 size; \ + size = (_ring).size; \ +- if ((ZXDH_RING_USED_QUANTA(_ring) + (_count)) < size) { \ +- (_ring).head = ((_ring).head + (_count)) % size; \ +- (_retcode) = 0; \ +- } else { \ +- (_retcode) = ZXDH_ERR_RING_FULL; \ +- } \ +- } +-#define ZXDH_SQ_RING_MOVE_HEAD(_ring, _retcode) \ +- { \ +- register __u32 size; \ +- size = (_ring).size; \ +- if (!ZXDH_SQ_RING_FULL_ERR(_ring)) { \ +- (_ring).head = ((_ring).head + 1) % size; \ +- (_retcode) = 0; \ +- } else { \ +- (_retcode) = ZXDH_ERR_RING_FULL; \ +- } \ +- } +-#define ZXDH_SQ_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \ +- { \ +- register __u32 size; \ +- size = (_ring).size; \ +- if ((ZXDH_RING_USED_QUANTA(_ring) + (_count)) < \ +- (size - 256)) { \ ++ if ((_ring).used_quanta_cnt + (_count) <= size) { \ + (_ring).head = ((_ring).head + (_count)) % size; \ ++ (_ring).used_quanta_cnt += (_count); \ + (_retcode) = 0; \ + } else { \ + (_retcode) = ZXDH_ERR_RING_FULL; \ + } \ + } ++ + #define ZXDH_RING_MOVE_HEAD_BY_COUNT_NOCHECK(_ring, _count) \ +- (_ring).head = ((_ring).head + (_count)) % (_ring).size ++ ((_ring).head = ((_ring).head + (_count)) % (_ring).size, \ ++ (_ring).used_quanta_cnt += (_count)) + + #define ZXDH_RING_MOVE_TAIL(_ring) \ +- (_ring).tail = ((_ring).tail + 1) % (_ring).size ++ ((_ring).tail = ((_ring).tail + 1) % (_ring).size, \ ++ (_ring).used_quanta_cnt -= 1) + + #define ZXDH_RING_MOVE_HEAD_NOCHECK(_ring) \ +- (_ring).head = ((_ring).head + 1) % (_ring).size ++ ((_ring).head = ((_ring).head + 1) % (_ring).size, \ ++ (_ring).used_quanta_cnt += 1) + + #define ZXDH_RING_MOVE_TAIL_BY_COUNT(_ring, _count) \ +- (_ring).tail = ((_ring).tail + (_count)) % (_ring).size ++ ((_ring).tail = ((_ring).tail + (_count)) % (_ring).size, \ ++ (_ring).used_quanta_cnt -= (_count)) + + #define ZXDH_RING_SET_TAIL(_ring, _pos) (_ring).tail = (_pos) % (_ring).size + + #define ZXDH_RING_FULL_ERR(_ring) \ +- ((ZXDH_RING_USED_QUANTA(_ring) == ((_ring).size - 1))) ++ ((_ring).used_quanta_cnt == (_ring).size) + + #define ZXDH_ERR_RING_FULL2(_ring) \ + ((ZXDH_RING_USED_QUANTA(_ring) == ((_ring).size - 2))) +@@ -298,23 +302,13 @@ + #define ZXDH_ERR_RING_FULL3(_ring) \ + ((ZXDH_RING_USED_QUANTA(_ring) == ((_ring).size - 3))) + +-#define ZXDH_SQ_RING_FULL_ERR(_ring) \ +- ((ZXDH_RING_USED_QUANTA(_ring) == ((_ring).size - 257))) +- +-#define ZXDH_ERR_SQ_RING_FULL2(_ring) \ +- ((ZXDH_RING_USED_QUANTA(_ring) == ((_ring).size - 258))) +-#define ZXDH_ERR_SQ_RING_FULL3(_ring) \ +- ((ZXDH_RING_USED_QUANTA(_ring) == ((_ring).size - 259))) +-#define ZXDH_RING_MORE_WORK(_ring) ((ZXDH_RING_USED_QUANTA(_ring) != 0)) ++#define ZXDH_RING_MORE_WORK(_ring) (((_ring).used_quanta_cnt != 0)) + + #define ZXDH_RING_USED_QUANTA(_ring) \ +- ((((_ring).head + (_ring).size - (_ring).tail) % (_ring).size)) ++ ((_ring).used_quanta_cnt) + + #define ZXDH_RING_FREE_QUANTA(_ring) \ +- (((_ring).size - ZXDH_RING_USED_QUANTA(_ring) - 1)) +- +-#define ZXDH_SQ_RING_FREE_QUANTA(_ring) \ +- (((_ring).size - ZXDH_RING_USED_QUANTA(_ring) - 257)) ++ ((_ring).size - (_ring).used_quanta_cnt) + + #define ZXDH_ATOMIC_RING_MOVE_HEAD(_ring, index, _retcode) \ + { \ +@@ -322,6 +316,7 @@ + ZXDH_RING_MOVE_HEAD(_ring, _retcode); \ + } + ++ + enum zxdh_qp_wqe_size { + ZXDH_WQE_SIZE_32 = 32, + ZXDH_WQE_SIZE_64 = 64, +@@ -376,6 +371,18 @@ + } + + /** ++ * get_64bit_volatile_val - read 64 bit volatile value from wqe ++ * @wqe_words: wqe addr ++ * @byte_index: index to read from ++ * @val: read value (volatile pointer) ++ **/ ++static inline void get_64bit_volatile_val(__le64 *wqe_words, __u32 byte_index, ++ volatile __u64 *val) ++{ ++ *val = le64toh(wqe_words[byte_index >> 3]); ++} ++ ++/** + * get_32bit_val - read 32 bit value from wqe + * @wqe_words: wqe addr + * @byte_index: index to reaad from +@@ -392,8 +399,8 @@ + *wqe_word = val; + } + +-#define read_wqe_need_split(pre_cal_psn, next_psn) \ +- (((pre_cal_psn < next_psn) && (pre_cal_psn != 0)) || \ +- ((next_psn <= 0x7FFFFF) && (pre_cal_psn > 0x800000))) +- +-#endif /* ZXDH_DEFS_H */ ++#define read_wqe_need_split(pre_cal_psn, next_psn, chip_rev) \ ++ (!(chip_rev == 2) && \ ++ (((pre_cal_psn < next_psn) && (pre_cal_psn != 0)) || \ ++ ((next_psn <= 0x7FFFFF) && (pre_cal_psn > 0x800000)))) ++#endif /* ZXDH_DEFS_H */ +\ 文件尾没有换行符 +diff -ruN baseline/providers/zrdma/zxdh_devids.h b/providers/zrdma/zxdh_devids.h +--- a/providers/zrdma/zxdh_devids.h 2026-08-06 15:37:55.339861870 +0800 ++++ b/providers/zrdma/zxdh_devids.h 2026-08-06 15:29:37.253676126 +0800 +@@ -6,12 +6,80 @@ + /* ZXDH VENDOR ID */ + #define PCI_VENDOR_ID_ZXDH_EVB 0x16c3 + #define PCI_VENDOR_ID_ZXDH_E312 0x1cf2 ++#define PCI_VENDOR_ID_ZXDH_E312S_D 0x1cf2 ++#define PCI_VENDOR_ID_ZXDH_E310 0x1cf2 ++#define PCI_VENDOR_ID_ZXDH_E310_RDMA 0x1cf2 ++#define PCI_VENDOR_ID_ZXDH_E316 0x1cf2 ++#define PCI_VENDOR_ID_ZXDH_E316L 0x1cf2 ++#define PCI_VENDOR_ID_ZXDH_E318 0x1cf2 + #define PCI_VENDOR_ID_ZXDH_X512 0x1cf2 ++#define PCI_VENDOR_ID_ZXDH_E312_TY_CLOUD 0x1b18 ++#define PCI_VENDOR_ID_ZXDH_E310_TY_CLOUD 0x1b18 ++#define PCI_VENDOR_ID_ZXDH_E312_XUANWU 0x8460 ++#define PCI_VENDOR_ID_ZXDH_E316_XUANWU 0x8460 ++#define PCI_VENDOR_ID_ZXDH_E318_XUANWU 0x8460 ++#define PCI_VENDOR_ID_ZXDH_N25HX2F200G 0x8848 ++#define PCI_VENDOR_ID_ZXDH_N25HX1F400G 0x8848 ++ ++#define PCI_VENDOR_ID_ZXDH_1X200 0x8848 ++#define PCI_VENDOR_ID_ZXDH_E316_XUANWU_LX 0x1234 ++#define PCI_VENDOR_ID_ZXDH_E318_XUANWU_LX 0x1234 ++#define PCI_VENDOR_ID_ZXDH_E310_RDMA_XUANWU_YZ 0x6798 ++#define PCI_VENDOR_ID_ZXDH_E312_RDMA_XUANWU_YZ 0x6798 ++#define PCI_VENDOR_ID_ZXDH_E312_S_XUANWU_YZ 0x6798 ++#define PCI_VENDOR_ID_ZXDH_E316_L_XUANWU_YZ 0x6798 ++#define PCI_VENDOR_ID_ZXDH_E316_XUANWU_YZ 0x6798 ++#define PCI_VENDOR_ID_ZXDH_E318_XUANWU_YZ 0x6798 ++ + /* ZXDH Devices ID */ + #define ZXDH_DEV_ID_ADAPTIVE_EVB_PF 0x8040 /* ZXDH EVB PF DEVICE ID*/ + #define ZXDH_DEV_ID_ADAPTIVE_EVB_VF 0x8041 /* ZXDH EVB VF DEVICE ID*/ + #define ZXDH_DEV_ID_ADAPTIVE_E312_PF 0x8049 /* ZXDH E312 PF DEVICE ID*/ + #define ZXDH_DEV_ID_ADAPTIVE_E312_VF 0x8060 /* ZXDH E312 VF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E310_PF 0x8061 /* ZXDH E310 PF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E310_VF 0x8062 /* ZXDH E310 VF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E310_RDMA_PF 0x8084 /* ZXDH E310_RDMA PF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E310_RDMA_VF 0x8085 /* ZXDH E310_RDMA VF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E316_PF 0x807e /* ZXDH E316 PF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E316_VF 0x807f /* ZXDH E316 VF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E316L_PF 0x80bd /* ZXDH E316L PF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E316L_VF 0x80be /* ZXDH E316L VF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E318_PF 0x80ab /* ZXDH E318 PF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E318_VF 0x80ac /* ZXDH E318 VF DEVICE ID*/ + #define ZXDH_DEV_ID_ADAPTIVE_X512_PF 0x806B /* ZXDH X512 PF DEVICE ID*/ + #define ZXDH_DEV_ID_ADAPTIVE_X512_VF 0x806C /* ZXDH X512 VF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E312_TY_CLOUD_PF 0x1110 /* ZXDH E312 TY CLOUD PF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E312_TY_CLOUD_VF 0x1111 /* ZXDH E312 TY CLOUD VF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E310_TY_CLOUD_PF 0x1100 /* ZXDH E310 TY CLOUD PF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E310_TY_CLOUD_VF 0x1101 /* ZXDH E310 TY CLOUD VF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E312S_D_PF 0x80a2 /* ZXDH E310 TY CLOUD PF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E312S_D_VF 0x80a3 /* ZXDH E310 TY CLOUD VF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E312_XUANWU_PF 0x0205 /* ZXDH E312 XUANWU PF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E312_XUANWU_VF 0x0206 /* ZXDH E312 XUANWU VF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E316_XUANWU_PF 0x0207 /* ZXDH E316 XUANWU PF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E316_XUANWU_VF 0x0208 /* ZXDH E316 XUANWU VF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E318_XUANWU_PF 0x0209 /* ZXDH E318 XUANWU PF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E318_XUANWU_VF 0x020A /* ZXDH E318 XUANWU VF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_1X200_PF 0x8607 /* ZXDH 1X200 PF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_1X200_VF 0x8608 /* ZXDH 1X200 VF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E316_XUANWU_LX_PF 0xfff2 /* ZXDH E316 XUANWU LX PF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E316_XUANWU_LX_VF 0xfff3 /* ZXDH E316 XUANWU LX VF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E318_XUANWU_LX_PF 0xfff4 /* ZXDH E318 XUANWU LX PF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E318_XUANWU_LX_VF 0xfff5 /* ZXDH E318 XUANWU LX VF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_N25HX2F200G_PF 0x8601 /* ZXDH N25HX2F200G PF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_N25HX2F200G_VF 0x8602 /* ZXDH N25HX2F200G VF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_N25HX1F400G_PF 0x8603 /* ZXDH N25HX1F400G PF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_N25HX1F400G_VF 0x8604 /* ZXDH N25HX1F400G VF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E310_RDMA_XUANWU_YZ_PF 0xffa2 /* ZXDH E310_RDMA XUANWU YZ PF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E310_RDMA_XUANWU_YZ_VF 0xffa3 /* ZXDH E310_RDMA XUANWU YZ VF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E312_RDMA_XUANWU_YZ_PF 0xffb2 /* ZXDH E312_RDMA XUANWU YZ PF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E312_RDMA_XUANWU_YZ_VF 0xffb3 /* ZXDH E312_RDMA XUANWU YZ VF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E312_S_XUANWU_YZ_PF 0xffc2 /* ZXDH E312_S XUANWU YZ PF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E312_S_XUANWU_YZ_VF 0xffc3 /* ZXDH E312_S XUANWU YZ VF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E316_L_XUANWU_YZ_PF 0xffd2 /* ZXDH E316_L XUANWU YZ PF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E316_L_XUANWU_YZ_VF 0xffd3 /* ZXDH E316_L XUANWU YZ VF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E316_XUANWU_YZ_PF 0xffe2 /* ZXDH E316 XUANWU YZ PF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E316_XUANWU_YZ_VF 0xffe3 /* ZXDH E316 XUANWU YZ VF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E318_XUANWU_YZ_PF 0xfff6 /* ZXDH E318 XUANWU YZ PF DEVICE ID*/ ++#define ZXDH_DEV_ID_ADAPTIVE_E318_XUANWU_YZ_VF 0xfff7 /* ZXDH E318 XUANWU YZ VF DEVICE ID*/ + #endif /* ZXDH_DEVIDS_H */ +diff -ruN baseline/providers/zrdma/zxdh_devx.c b/providers/zrdma/zxdh_devx.c +--- a/providers/zrdma/zxdh_devx.c 1970-01-01 08:00:00.000000000 +0800 ++++ b/providers/zrdma/zxdh_devx.c 2026-08-06 15:29:36.607631759 +0800 +@@ -0,0 +1,941 @@ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "zxdh_zrdma.h" ++#include "zxdh_abi.h" ++#include "zxdh_devx.h" ++ ++static inline int get_cq_size(int ncqe) ++{ ++ if (ncqe < ZXDH_U_MINCQ_SIZE) ++ ncqe = ZXDH_U_MINCQ_SIZE; ++ return ncqe; ++} ++ ++static inline size_t get_cq_total_bytes(__u32 cq_size) ++{ ++ return roundup(cq_size * ZXDH_CQ_CQE_SIZE, ZXDH_HW_PAGE_SIZE); ++} ++ ++static int zxdh_get_validated_cq_size(struct zxdh_uvcontext *iwvctx, ++ int requested_cqe, __u32 *cq_size) ++{ ++ struct zxdh_dev_attrs *dev_attrs; ++ dev_attrs = &iwvctx->dev_attrs; ++ ++ if (requested_cqe < ZXDH_MIN_CQ_SIZE || ++ requested_cqe > dev_attrs->max_hw_cq_size) { ++ errno = EINVAL; ++ return errno; ++ } ++ ++ *cq_size = get_cq_size(requested_cqe); ++ *cq_size = zxdh_cq_round_up(*cq_size); ++ if (*cq_size > dev_attrs->max_hw_cq_size) { ++ errno = EINVAL; ++ return errno; ++ } ++ ++ return 0; ++} ++ ++static void zxdh_calculate_cq_memory_size(__u32 cq_size, __u32 *cq_pages, ++ size_t *total_size) ++{ ++ size_t cq_bytes; ++ size_t shadow_size; ++ ++ cq_bytes = get_cq_total_bytes(cq_size); ++ *cq_pages = cq_bytes >> ZXDH_HW_PAGE_SHIFT; ++ shadow_size = roundup(ZXDH_DB_SHADOW_AREA_SIZE, ZXDH_HW_PAGE_SIZE); ++ *total_size = (*cq_pages << ZXDH_HW_PAGE_SHIFT) + shadow_size; ++} ++ ++static int zxdh_devx_cq_init(struct zxdh_udevx_cq *cq, ++ struct zxdh_udevx_cq_init_info *info) ++{ ++ cq->devx_cq.verbs_cq.cq.cqe = info->cq_size; ++ cq->cq_base = info->cq_base; ++ cq->cq_id = info->cq_id; ++ cq->cq_size = info->cq_size; ++ cq->shadow_area = info->shadow_area; ++ cq->polarity = 1; ++ cq->devx_cq.cqn = info->cq_id; ++ cq->devx_cq.cqe_num = info->cq_size; ++ cq->devx_cq.polarity = 1; ++ cq->devx_cq.shadow_offset = info->shadow_offset; ++ cq->devx_cq.shadow_size = info->shadow_size; ++ return 0; ++} ++ ++int _zxdh_devx_get_cq_init_size(struct ibv_context *context, ++ struct zxdh_devx_cq_init_attr *attr, ++ struct zxdh_devx_init_cq_resp *resp) ++{ ++ size_t total_size; ++ __u32 cq_pages, cq_size; ++ int ret; ++ struct zxdh_uvcontext *iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ ++ ret = zxdh_get_validated_cq_size(iwvctx, attr->cqe_num, &cq_size); ++ if (ret) ++ return ret; ++ ++ zxdh_calculate_cq_memory_size(cq_size, &cq_pages, &total_size); ++ ++ resp->cqe_num = cq_size; ++ resp->total_size = total_size; ++ resp->shadow_offset = cq_pages << ZXDH_HW_PAGE_SHIFT; ++ return 0; ++} ++ ++static int zxdh_devx_modify_cq_overflow_check_en(struct zxdh_devx_cq *devx_cq, __u32 overflow_check_en) ++{ ++ int ret = 0; ++ ++ if (!devx_cq) { ++ return -EINVAL; ++ } ++ ++ ret = zxdh_devx_modify_cq(&devx_cq->verbs_cq.cq, overflow_check_en); ++ if (ret) ++ return ret; ++ ++ return 0; ++} ++ ++struct zxdh_devx_cq *_zxdh_devx_create_cq(struct ibv_context *context, ++ int cqe_num, ++ struct ibv_comp_channel *channel, ++ int comp_vector, ++ struct zxdh_devx_cq_buffers *buffer ++ ++) ++{ ++ struct zxdh_udevx_cq_init_info info = {}; ++ struct zxdh_ureg_mr reg_mr_cmd = {}; ++ struct zxdh_ureg_mr reg_mr_shadow_cmd = {}; ++ struct zxdh_ucreate_cq cmd = {}; ++ struct zxdh_ucreate_cq_resp resp = {}; ++ struct ib_uverbs_reg_mr_resp reg_mr_resp = {}; ++ struct ib_uverbs_reg_mr_resp reg_mr_shadow_resp = {}; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_udevx_cq *udevx_cq; ++ __u32 cq_pages; ++ size_t total_size; ++ int ret; ++ __u32 overflow_check_en = 0; ++ ++ iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ if (!buffer || !buffer->cq_base) { ++ errno = EINVAL; ++ return NULL; ++ } ++ if (channel || comp_vector) { ++ errno = EINVAL; ++ return NULL; ++ } ++ ret = zxdh_get_validated_cq_size(iwvctx, cqe_num, &info.cq_size); ++ if (ret) { ++ errno = EINVAL; ++ return NULL; ++ } ++ zxdh_calculate_cq_memory_size(info.cq_size, &cq_pages, &total_size); ++ if (buffer->total_size < total_size) { ++ errno = EINVAL; ++ return NULL; ++ } ++ info.cq_base = buffer->cq_base; ++ info.shadow_offset = cq_pages << ZXDH_HW_PAGE_SHIFT; ++ info.shadow_size = ZXDH_DB_SHADOW_AREA_SIZE; ++ ++ udevx_cq = calloc(1, sizeof(*udevx_cq)); ++ if (!udevx_cq) ++ return NULL; ++ ++ udevx_cq->comp_vector = comp_vector; ++ udevx_cq->buf_size = buffer->total_size; ++ reg_mr_cmd.reg_type = ZXDH_MEMREG_TYPE_CQ; ++ reg_mr_cmd.cq_pages = cq_pages; ++ ++ ret = ibv_cmd_reg_mr(&iwvctx->iwupd->ibv_pd, info.cq_base, ++ cq_pages << ZXDH_HW_PAGE_SHIFT, ++ (uintptr_t)info.cq_base, IBV_ACCESS_LOCAL_WRITE, ++ &udevx_cq->vmr, ®_mr_cmd.ibv_cmd, ++ sizeof(reg_mr_cmd), ®_mr_resp, ++ sizeof(reg_mr_resp)); ++ if (ret) { ++ errno = ret; ++ goto err_dereg_mr; ++ } ++ ++ udevx_cq->vmr.ibv_mr.pd = &iwvctx->iwupd->ibv_pd; ++ udevx_cq->devx_cq.verbs_cq.cq.context = context; ++ ++ info.shadow_area = (void *)(info.cq_base + info.shadow_offset); ++ ++ reg_mr_shadow_cmd.reg_type = ZXDH_MEMREG_TYPE_CQ; ++ reg_mr_shadow_cmd.cq_pages = 1; ++ ++ ret = ibv_cmd_reg_mr(&iwvctx->iwupd->ibv_pd, info.shadow_area, ++ ZXDH_HW_PAGE_SIZE, (uintptr_t)info.shadow_area, ++ IBV_ACCESS_LOCAL_WRITE, &udevx_cq->vmr_shadow_area, ++ ®_mr_shadow_cmd.ibv_cmd, ++ sizeof(reg_mr_shadow_cmd), ®_mr_shadow_resp, ++ sizeof(reg_mr_shadow_resp)); ++ if (ret) { ++ errno = ret; ++ goto err_dereg_shadow; ++ } ++ ++ udevx_cq->vmr_shadow_area.ibv_mr.pd = &iwvctx->iwupd->ibv_pd; ++ ++ cmd.user_cq_buf = (__u64)((uintptr_t)info.cq_base); ++ cmd.user_shadow_area = (__u64)((uintptr_t)info.shadow_area); ++ ++ ret = ibv_cmd_create_cq(context, info.cq_size, channel, comp_vector, ++ &udevx_cq->devx_cq.verbs_cq.cq, &cmd.ibv_cmd, ++ sizeof(cmd), &resp.ibv_resp, sizeof(resp)); ++ if (ret) { ++ errno = ret; ++ goto err_dereg_shadow; ++ } ++ info.cq_id = resp.cq_id; ++ ++ ret = zxdh_devx_modify_cq_overflow_check_en(&udevx_cq->devx_cq, overflow_check_en); ++ if (ret) { ++ errno = ret; ++ goto err_dereg_shadow; ++ } ++ ++ zxdh_devx_cq_init(udevx_cq, &info); ++ return &udevx_cq->devx_cq; ++ ++err_dereg_shadow: ++ ibv_cmd_dereg_mr(&udevx_cq->vmr); ++err_dereg_mr: ++ free(udevx_cq); ++ return NULL; ++} ++ ++int _zxdh_devx_destroy_cq(struct zxdh_devx_cq *cq) ++{ ++ struct zxdh_udevx_cq *udevx_cq; ++ int ret; ++ ++ if (!cq) { ++ return -EINVAL; ++ } ++ ++ udevx_cq = container_of(cq, struct zxdh_udevx_cq, devx_cq); ++ ++ ret = ibv_cmd_destroy_cq(&cq->verbs_cq.cq); ++ if (ret) ++ goto err; ++ ++ ibv_cmd_dereg_mr(&udevx_cq->vmr); ++ ++ free(udevx_cq); ++ return 0; ++ ++err: ++ return ret; ++} ++ ++int _zxdh_devx_get_qp_init_size(struct ibv_context *ibv_ctx, ++ struct zxdh_devx_qp_init_attr *attr, ++ struct zxdh_devx_init_qp_resp *resp) ++{ ++ int status; ++ size_t sqsize, rqsize, shadow_size, totalqpsize; ++ __u32 sqdepth, rqdepth; ++ __u8 sqshift, rqshift; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ ++ if (!ibv_ctx || !attr || !resp) { ++ return -EINVAL; ++ } ++ ++ if (attr->qp_type != IBV_QPT_RC) { ++ return -EOPNOTSUPP; ++ } ++ ++ iwvctx = container_of(ibv_ctx, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ ++ if (attr->cap.max_send_sge > dev_attrs->max_hw_wq_frags || ++ attr->cap.max_recv_sge > dev_attrs->max_hw_wq_frags) { ++ return -EINVAL; ++ } ++ ++ if (attr->cap.max_inline_data > dev_attrs->max_hw_inline) { ++ zxdh_dbg(ZXDH_DBG_QP, "max_inline_data over max_hw_inline\n"); ++ attr->cap.max_inline_data = dev_attrs->max_hw_inline; ++ } ++ ++ zxdh_get_sq_wqe_shift(attr->cap.max_send_sge, attr->cap.max_inline_data, ++ &sqshift); ++ status = zxdh_devx_get_sqdepth(dev_attrs, attr->cap.max_send_wr, sqshift, ++ &sqdepth); ++ if (status) { ++ return -EINVAL; ++ } ++ ++ zxdh_get_rq_wqe_shift(attr->cap.max_recv_sge, &rqshift); ++ status = zxdh_devx_get_rqdepth(dev_attrs, attr->cap.max_recv_wr, rqshift, ++ &rqdepth); ++ if (status) { ++ return -EINVAL; ++ } ++ ++ sqsize = roundup(sqdepth * ZXDH_QP_SQE_MIN_SIZE, ZXDH_HW_PAGE_SIZE); ++ rqsize = roundup((rqdepth << rqshift) * ZXDH_QP_RQE_MIN_SIZE, ZXDH_HW_PAGE_SIZE); ++ shadow_size = roundup(ZXDH_DB_SHADOW_AREA_SIZE, ZXDH_HW_PAGE_SIZE); ++ totalqpsize = rqsize + sqsize + shadow_size; ++ ++ resp->total_size = totalqpsize; ++ resp->rq_offset = sqsize; ++ resp->shadow_offset = sqsize + rqsize; ++ return 0; ++} ++ ++static int zxdh_devx_vmapped_qp(struct zxdh_udevx_qp *devx_uqp, ++ struct ibv_pd *pd, ++ struct ibv_qp_init_attr *attr, ++ struct zxdh_qp_init_info *info, ++ struct zxdh_devx_qp_buffers *buffer) ++{ ++ struct zxdh_ucreate_qp cmd = {}; ++ struct zxdh_ucreate_qp_resp resp = {}; ++ struct zxdh_ureg_mr reg_mr_cmd = {}; ++ struct ib_uverbs_reg_mr_resp reg_mr_resp = {}; ++ int status; ++ size_t sqsize = 0, rqsize = 0, shadow_size; ++ devx_uqp->buf_size = buffer->total_size; ++ ++ sqsize = roundup(info->sqdepth * ZXDH_QP_SQE_MIN_SIZE, ++ ZXDH_HW_PAGE_SIZE); ++ if (info->rqdepth > 0) { ++ rqsize = roundup((info->rqdepth << info->rqshift) * ZXDH_QP_RQE_MIN_SIZE, ++ ZXDH_HW_PAGE_SIZE); ++ reg_mr_cmd.rq_pages = rqsize >> ZXDH_HW_PAGE_SHIFT; ++ } ++ shadow_size = roundup(ZXDH_DB_SHADOW_AREA_SIZE, ZXDH_HW_PAGE_SIZE); ++ if ((sqsize + rqsize + shadow_size) != buffer->total_size) { ++ return -EINVAL; ++ } ++ ++ reg_mr_cmd.reg_type = ZXDH_MEMREG_TYPE_QP; ++ reg_mr_cmd.sq_pages = sqsize >> ZXDH_HW_PAGE_SHIFT; ++ status = ibv_cmd_reg_mr(pd, buffer->qp_base, buffer->total_size, ++ (uintptr_t)buffer->qp_base, ++ IBV_ACCESS_LOCAL_WRITE, &devx_uqp->vmr, ++ ®_mr_cmd.ibv_cmd, sizeof(reg_mr_cmd), ++ ®_mr_resp, sizeof(reg_mr_resp)); ++ if (status) ++ return status; ++ ++ cmd.user_wqe_bufs = (__u64)((uintptr_t)buffer->qp_base); ++ cmd.user_compl_ctx = (__u64)(uintptr_t)&devx_uqp->devx_qp; ++ ++ status = ibv_cmd_create_qp(pd, &devx_uqp->devx_qp.verbs_qp.qp, attr, ++ &cmd.ibv_cmd, sizeof(cmd), &resp.ibv_resp, ++ sizeof(struct zxdh_ucreate_qp_resp)); ++ if (status) { ++ goto err_qp; ++ } ++ ++ info->sq_size = resp.actual_sq_size; ++ info->rq_size = resp.actual_rq_size; ++ info->qp_caps = resp.qp_caps; ++ info->qp_id = resp.qp_id; ++ devx_uqp->zxdh_drv_opt = resp.zxdh_drv_opt; ++ devx_uqp->devx_qp.verbs_qp.qp.qp_num = resp.qp_id; ++ ++ if (!attr->send_cq || !attr->recv_cq) { ++ status = -EINVAL; ++ goto err_qp; ++ } ++ ++ devx_uqp->send_cq = ++ container_of(attr->send_cq, struct zxdh_udevx_cq, devx_cq.verbs_cq.cq); ++ devx_uqp->recv_cq = ++ container_of(attr->recv_cq, struct zxdh_udevx_cq, devx_cq.verbs_cq.cq); ++ if (devx_uqp->send_cq) { ++ devx_uqp->send_cq->uqp = devx_uqp; ++ } ++ if (devx_uqp->recv_cq) { ++ devx_uqp->recv_cq->uqp = devx_uqp; ++ } ++ ++ return 0; ++err_qp: ++ ibv_cmd_dereg_mr(&devx_uqp->vmr); ++ return status; ++} ++ ++enum zxdh_status_code zxdh_devx_qp_init(struct zxdh_udevx_qp *qp, ++ struct zxdh_qp_init_info *info) ++{ ++ enum zxdh_status_code ret_code = 0; ++ qp->qp_caps = info->qp_caps; ++ qp->qp_id = info->qp_id; ++ qp->sq_size = info->sq_size; ++ qp->rq_size = info->rq_size; ++ qp->qp_type = info->type; ++ qp->devx_qp.wqe_alloc_db = info->wqe_alloc_db; ++ qp->devx_qp.qpn = info->qp_id; ++ qp->devx_qp.sqdepth = info->sqdepth; ++ qp->devx_qp.sqshift = info->sqshift; ++ qp->devx_qp.rqdepth = info->rqdepth; ++ qp->devx_qp.rqshift = info->rqshift; ++ qp->devx_qp.sq_polarity = 1; ++ qp->devx_qp.rq_polarity = 0; ++ qp->devx_qp.max_sq_frag_cnt = info->max_sq_frag_cnt; ++ qp->devx_qp.max_rq_frag_cnt = info->max_rq_frag_cnt; ++ qp->devx_qp.max_inline_data = (info->max_inline_data == 0) ? ++ ZXDH_MAX_INLINE_DATA_SIZE : ++ info->max_inline_data; ++ qp->devx_qp.rq_wqe_size_multiplier = 1 << info->rqshift; ++ qp->devx_qp.shadow_init_value = 0x8000; ++ qp->devx_qp.shadow_size = ZXDH_DB_SHADOW_AREA_SIZE; ++ return ret_code; ++} ++ ++struct zxdh_devx_qp *_zxdh_devx_create_qp(struct ibv_pd *pd, ++ struct ibv_qp_init_attr *attr, ++ struct zxdh_devx_qp_buffers *buffer) ++{ ++ struct zxdh_qp_init_info info = { 0 }; ++ struct zxdh_uvcontext *iwvctx; ++ int status; ++ struct zxdh_dev_attrs *dev_attrs; ++ struct zxdh_udevx_qp *devx_uqp; ++ struct ibv_context *ibv_ctx; ++ ++ if (!pd || !attr || !buffer) { ++ errno = EINVAL; ++ return NULL; ++ } ++ ++ if (buffer->qp_base == NULL || buffer->total_size == 0) { ++ errno = EINVAL; ++ return NULL; ++ } ++ if (attr->qp_type != IBV_QPT_RC) { ++ errno = EOPNOTSUPP; ++ return NULL; ++ } ++ ibv_ctx = pd->context; ++ iwvctx = container_of(ibv_ctx, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ if (attr->cap.max_send_sge > dev_attrs->max_hw_wq_frags || ++ attr->cap.max_recv_sge > dev_attrs->max_hw_wq_frags) { ++ errno = EINVAL; ++ return NULL; ++ } ++ ++ if (attr->cap.max_inline_data > dev_attrs->max_hw_inline) { ++ zxdh_dbg(ZXDH_DBG_QP, "max_inline_data over max_hw_inline\n"); ++ attr->cap.max_inline_data = dev_attrs->max_hw_inline; ++ } ++ ++ zxdh_get_sq_wqe_shift(attr->cap.max_send_sge, attr->cap.max_inline_data, ++ &info.sqshift); ++ status = zxdh_devx_get_sqdepth(dev_attrs, attr->cap.max_send_wr, ++ info.sqshift, &info.sqdepth); ++ if (status) { ++ errno = EINVAL; ++ return NULL; ++ } ++ ++ zxdh_get_rq_wqe_shift(attr->cap.max_recv_sge, &info.rqshift); ++ status = zxdh_devx_get_sqdepth(dev_attrs, attr->cap.max_recv_wr, ++ info.rqshift, &info.rqdepth); ++ if (status) { ++ errno = EINVAL; ++ return NULL; ++ } ++ ++ devx_uqp = memalign(1024, sizeof(*devx_uqp)); ++ if (!devx_uqp) { ++ errno = ENOMEM; ++ return NULL; ++ } ++ memset(devx_uqp, 0, sizeof(*devx_uqp)); ++ ++ devx_uqp->devx_qp.sq_sig_all = attr->sq_sig_all; ++ devx_uqp->devx_qp.verbs_qp.qp.qp_type = attr->qp_type; ++ devx_uqp->devx_qp.verbs_qp.qp.context = ibv_ctx; ++ info.sq_size = info.sqdepth >> info.sqshift; ++ info.rq_size = info.rqdepth; ++ attr->cap.max_send_wr = info.sq_size; ++ attr->cap.max_recv_wr = info.rq_size; ++ ++ info.dev_attrs = dev_attrs; ++ info.max_sq_frag_cnt = attr->cap.max_send_sge; ++ info.max_rq_frag_cnt = attr->cap.max_recv_sge; ++ ++ info.wqe_alloc_db = ++ (__u32 *)((__u8 *)iwvctx->sq_db + ZXDH_DB_SQ_OFFSET); ++ info.abi_ver = iwvctx->abi_ver; ++ info.legacy_mode = iwvctx->legacy_mode; ++ ++ info.type = ZXDH_QP_TYPE_ROCE_RC; ++ status = zxdh_devx_vmapped_qp(devx_uqp, pd, attr, &info, buffer); ++ if (status) { ++ errno = status; ++ goto err_free_qp; ++ } ++ ++ info.max_sq_frag_cnt = attr->cap.max_send_sge; ++ info.max_rq_frag_cnt = attr->cap.max_recv_sge; ++ info.max_inline_data = attr->cap.max_inline_data; ++ ++ zxdh_devx_qp_init(devx_uqp, &info); ++ attr->cap.max_send_wr = (info.sqdepth - ZXDH_SQ_RSVD) >> info.sqshift; ++ attr->cap.max_recv_wr = info.rqdepth - ZXDH_RQ_RSVD; ++ ++ return &devx_uqp->devx_qp; ++ ++err_free_qp: ++ free(devx_uqp); ++ return NULL; ++} ++ ++static int zxdh_destroy_devx_vmapped_qp(struct zxdh_udevx_qp *devx_uqp) ++{ ++ int ret; ++ ++ ret = ibv_cmd_destroy_qp(&devx_uqp->devx_qp.verbs_qp.qp); ++ if (ret) ++ return ret; ++ ++ ibv_cmd_dereg_mr(&devx_uqp->vmr); ++ ++ return 0; ++} ++ ++int _zxdh_devx_destroy_qp(struct zxdh_devx_qp *devx_qp) ++{ ++ int ret; ++ struct zxdh_udevx_qp *devx_uqp; ++ ++ if (!devx_qp) { ++ return -EINVAL; ++ } ++ ++ devx_uqp = container_of(devx_qp, struct zxdh_udevx_qp, devx_qp); ++ ++ devx_uqp->destroy_pending = true; ++ ++ ret = zxdh_destroy_devx_vmapped_qp(devx_uqp); ++ if (ret) ++ goto err; ++ free(devx_uqp); ++ ++ return 0; ++ ++err: ++ return ret; ++} ++ ++static void zxdh_init_qp_indices(struct zxdh_udevx_qp *devx_uqp) ++{ ++ devx_uqp->devx_qp.sq_polarity = 0; ++ devx_uqp->devx_qp.rq_polarity = 0; ++ devx_uqp->devx_qp.rq_signature = 0; ++} ++ ++int _zxdh_devx_modify_qp(struct zxdh_devx_qp *devx_qp, struct ibv_qp_attr *attr, ++ int attr_mask) ++{ ++ struct zxdh_udevx_qp *devx_uqp; ++ struct zxdh_umodify_qp_resp resp = {}; ++ struct ibv_modify_qp cmd = {}; ++ struct zxdh_umodify_qp cmd_ex = {}; ++ int ret = 0; ++ __u16 mtu = 0; ++ ++ if (!devx_qp || !attr) { ++ return -EINVAL; ++ } ++ ++ devx_uqp = container_of(devx_qp, struct zxdh_udevx_qp, devx_qp); ++ if (!ret && (attr_mask & IBV_QP_PATH_MTU) && ++ devx_qp->verbs_qp.qp.qp_type == IBV_QPT_RC) { ++ mtu = mtu_enum_to_int(attr->path_mtu); ++ if (mtu == 0) ++ return -EINVAL; ++ } ++ if (attr_mask & IBV_QP_STATE || attr_mask & IBV_QP_RATE_LIMIT) { ++ ret = ibv_cmd_modify_qp_ex(&devx_qp->verbs_qp.qp, attr, ++ attr_mask, &cmd_ex.ibv_cmd, ++ sizeof(cmd_ex), &resp.ibv_resp, ++ sizeof(resp)); ++ } else { ++ ret = ibv_cmd_modify_qp(&devx_qp->verbs_qp.qp, attr, attr_mask, ++ &cmd, sizeof(cmd)); ++ } ++ ++ if (!ret && (attr_mask & IBV_QP_STATE) && ++ attr->qp_state == IBV_QPS_RESET) { ++ zxdh_init_qp_indices(devx_uqp); ++ } ++ ++ return ret; ++} ++ ++struct ibv_mr *_zxdh_devx_reg_mr(struct ibv_pd *pd, void *host_addr, size_t length, uint64_t dev_addr, unsigned int access) ++{ ++ struct zxdh_umr *umr; ++ struct zxdh_ureg_mr cmd; ++ struct zxdh_ureg_mr_resp resp = {}; ++ int err; ++ ++ umr = malloc(sizeof(*umr)); ++ if (!umr) ++ return NULL; ++ ++ cmd.reg_type = ZXDH_MEMREG_TYPE_EXTERNEL_MEM; ++ err = ibv_cmd_reg_mr(pd, host_addr, length, dev_addr, access, &umr->vmr, ++ &cmd.ibv_cmd, sizeof(cmd), &resp.ibv_resp, ++ sizeof(resp)); ++ if (err) { ++ free(umr); ++ errno = err; ++ return NULL; ++ } ++ ++ umr->acc_flags = access; ++ umr->host_page_size = resp.host_page_size; ++ return &umr->vmr.ibv_mr; ++} ++ ++int _zxdh_devx_dereg_mr(struct ibv_mr *mr) ++{ ++ struct verbs_mr *vmr = verbs_get_mr(mr); ++ int ret; ++ ++ ret = ibv_cmd_dereg_mr(vmr); ++ if (ret) ++ return ret; ++ ++ free(vmr); ++ return 0; ++} ++ ++static struct ibv_cq *ucreate_cq_with_ext_buf(struct ibv_context *context, ++ struct ibv_cq_init_attr_ex *attr_ex, ++ struct zxdh_ext_buf *buffer) ++{ ++ struct zxdh_cq_init_info info = {}; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_ucq *iwucq; ++ struct zxdh_ucreate_cq_ex cmd = {}; ++ struct zxdh_ucreate_cq_ex_resp resp = {}; ++ size_t total_size; ++ __u32 cq_pages, shadow_offset; ++ int ret, ncqe; ++ int lock_inited = 0; ++ ++ iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ ret = zxdh_get_validated_cq_size(iwvctx, attr_ex->cqe, &info.cq_size); ++ if (ret) { ++ errno = EINVAL; ++ return NULL; ++ } ++ ++ iwucq = calloc(1, sizeof(*iwucq)); ++ if (!iwucq) { ++ errno = ENOMEM; ++ return NULL; ++ } ++ memset(iwucq, 0, sizeof(*iwucq)); ++ ++ ret = pthread_spin_init(&iwucq->lock, PTHREAD_PROCESS_PRIVATE); ++ if (ret) { ++ errno = (ret > 0) ? ret : -ret; ++ goto err; ++ } ++ lock_inited = 1; ++ ++ iwucq->resize_enable = false; ++ iwucq->is_ext_buf_mode = true; ++ iwucq->comp_vector = attr_ex->comp_vector; ++ ++ zxdh_calculate_cq_memory_size(info.cq_size, &cq_pages, &total_size); ++ buffer->total_size = total_size; ++ iwucq->buf_size = total_size; ++ shadow_offset = cq_pages << ZXDH_HW_PAGE_SHIFT; ++ iwucq->vmr.ibv_mr.pd = &iwvctx->iwupd->ibv_pd; ++ info.cq_base = buffer->host_addr; ++ info.shadow_area = (__le64 *)((__u8 *)info.cq_base + shadow_offset); ++ ++ ncqe = attr_ex->cqe; ++ attr_ex->cqe = info.cq_size; ++ ++ cmd.user_cq_buf = (__u64)((uintptr_t)info.cq_base); ++ cmd.user_shadow_area = (__u64)((uintptr_t)info.shadow_area); ++ cmd.dev_cq_buf = buffer->dev_addr; ++ cmd.dev_shadow_area = buffer->dev_addr + shadow_offset; ++ ret = ibv_cmd_create_cq_ex_wrap(context, attr_ex, &iwucq->verbs_cq, &cmd.ibv_cmd, sizeof(cmd), &resp.ibv_resp, sizeof(resp)); ++ if (ret) { ++ errno = (ret > 0) ? ret : -ret; ++ goto err; ++ } ++ info.cq_id = resp.cq_id; ++ iwucq->verbs_cq.cq.cqe = ncqe; ++ info.cqe_alloc_db = (__u32*)((__u8*)iwvctx->cq_db + ZXDH_DB_CQ_OFFSET); ++ ++ zxdh_cq_init(&iwucq->cq, &info); ++ return &iwucq->verbs_cq.cq; ++err: ++ if (lock_inited) { ++ int __destroy_ret = pthread_spin_destroy(&iwucq->lock); ++ (void)__destroy_ret; ++ } ++ free(iwucq); ++ return NULL; ++} ++ ++struct ibv_cq *_zxdh_devx_create_cq_with_ext_buf(struct ibv_context *context, ++ int cqe, struct ibv_comp_channel *channel, ++ int comp_vector, struct zxdh_ext_buf *buf) ++{ ++ struct ibv_cq *ibv_cq; ++ struct ibv_cq_init_attr_ex attr_ex = { .cqe = cqe, .channel = channel, .comp_vector = comp_vector }; ++ ++ if (!buf || !buf->host_addr || !buf->dev_addr) { ++ errno = EINVAL; ++ return NULL; ++ } ++ ++ if (channel || comp_vector) { ++ errno = EINVAL; ++ return NULL; ++ } ++ ++ ibv_cq = ucreate_cq_with_ext_buf(context, &attr_ex, buf); ++ if (!ibv_cq) { ++ return NULL; ++ } ++ ++ return ibv_cq; ++} ++ ++static int zxdh_vmapped_qp_with_ext_buff(struct zxdh_uqp *iwuqp, ++ struct ibv_pd *pd, ++ struct ibv_qp_init_attr *attr, ++ struct zxdh_qp_init_info *info, ++ struct zxdh_ext_buf *buffer) ++{ ++ struct zxdh_ucreate_qp cmd = {}; ++ struct zxdh_ucreate_qp_resp resp = {}; ++ int ret; ++ size_t sqsize, rqsize, shadow_size; ++ ++ if (!attr->send_cq || !attr->recv_cq) { ++ return -EINVAL; ++ } ++ ++ sqsize = roundup(info->sqdepth * ZXDH_QP_SQE_MIN_SIZE, ZXDH_HW_PAGE_SIZE); ++ rqsize = roundup((info->rqdepth << info->rqshift) * ZXDH_QP_RQE_MIN_SIZE, ZXDH_HW_PAGE_SIZE); ++ shadow_size = roundup(ZXDH_DB_SHADOW_AREA_SIZE, ZXDH_HW_PAGE_SIZE); ++ buffer->total_size = sqsize + rqsize + shadow_size; ++ iwuqp->buf_size = buffer->total_size; ++ ++ info->sq = (struct zxdh_qp_sq_quanta *)buffer->host_addr; ++ info->rq = (struct zxdh_qp_rq_quanta *)((__u8 *)buffer->host_addr + sqsize); ++ info->shadow_area = (__le64 *)((__u8 *)buffer->host_addr + sqsize + rqsize); ++ ++ cmd.user_wqe_bufs = (__u64)((uintptr_t)info->sq); ++ cmd.user_compl_ctx = (__u64)(uintptr_t)&iwuqp->qp; ++ cmd.dev_wqe_bufs = buffer->dev_addr; ++ cmd.rq_offset = (__u32)sqsize; ++ cmd.shadow_offset = (__u32)(sqsize + rqsize); ++ ++ ret = ibv_cmd_create_qp(pd, &iwuqp->vqp.qp, attr, ++ &cmd.ibv_cmd, sizeof(cmd), &resp.ibv_resp, ++ sizeof(struct zxdh_ucreate_qp_resp)); ++ if (ret != 0) { ++ return ret; ++ } ++ ++ info->sq_size = resp.actual_sq_size; ++ info->rq_size = resp.actual_rq_size; ++ info->qp_caps = resp.qp_caps; ++ info->qp_id = resp.qp_id; ++ iwuqp->zxdh_drv_opt = resp.zxdh_drv_opt; ++ iwuqp->qp.qp_id = resp.qp_id; ++ iwuqp->vqp.qp.qp_num = resp.qp_id; ++ iwuqp->send_cq = container_of(attr->send_cq, struct zxdh_ucq, verbs_cq.cq); ++ iwuqp->recv_cq = container_of(attr->recv_cq, struct zxdh_ucq, verbs_cq.cq); ++ iwuqp->send_cq->uqp = iwuqp; ++ iwuqp->recv_cq->uqp = iwuqp; ++ ++ return 0; ++} ++ ++struct ibv_qp *_zxdh_devx_create_qp_with_ext_buf(struct ibv_pd *pd, ++ struct ibv_qp_init_attr *attr, ++ struct zxdh_ext_buf *buf) ++{ ++ struct zxdh_uqp *iwuqp; ++ struct zxdh_qp_init_info info = {}; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ int status; ++ ++ if (!pd || !attr || !buf || !buf->host_addr || !buf->dev_addr) { ++ errno = EINVAL; ++ return NULL; ++ } ++ ++ if (attr->qp_type != IBV_QPT_RC || attr->srq != NULL) { ++ errno = EOPNOTSUPP; ++ return NULL; ++ } ++ ++ iwvctx = container_of(pd->context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ ++ if (attr->cap.max_send_sge > dev_attrs->max_hw_wq_frags || ++ attr->cap.max_recv_sge > dev_attrs->max_hw_wq_frags) { ++ errno = EINVAL; ++ return NULL; ++ } ++ ++ if (attr->cap.max_inline_data > dev_attrs->max_hw_inline) { ++ zxdh_dbg(ZXDH_DBG_QP, "max_inline_data over max_hw_inline\n"); ++ attr->cap.max_inline_data = dev_attrs->max_hw_inline; ++ } ++ ++ zxdh_get_sq_wqe_shift(attr->cap.max_send_sge, attr->cap.max_inline_data, ++ &info.sqshift); ++ status = zxdh_get_sqdepth(dev_attrs, attr->cap.max_send_wr, ++ info.sqshift, &info.sqdepth); ++ if (status) { ++ errno = EINVAL; ++ return NULL; ++ } ++ ++ zxdh_get_rq_wqe_shift(attr->cap.max_recv_sge, &info.rqshift); ++ status = zxdh_get_rqdepth(dev_attrs, attr->cap.max_recv_wr, ++ info.rqshift, &info.rqdepth); ++ if (status) { ++ errno = EINVAL; ++ return NULL; ++ } ++ ++ iwuqp = memalign(1024, sizeof(*iwuqp)); ++ if (!iwuqp) { ++ errno = ENOMEM; ++ return NULL; ++ } ++ memset(iwuqp, 0, sizeof(*iwuqp)); ++ iwuqp->ext_buf_mode = 1; ++ if (pthread_spin_init(&iwuqp->lock, PTHREAD_PROCESS_PRIVATE)) { ++ errno = ENOMEM; ++ goto err_free_qp; ++ } ++ if (pthread_spin_init(&iwuqp->quanta_lock, PTHREAD_PROCESS_PRIVATE)) { ++ errno = ENOMEM; ++ goto err_destroy_quanta_lock; ++ } ++ ++ attr->cap.max_send_wr = info.sqdepth >> info.sqshift; ++ attr->cap.max_recv_wr = info.rqdepth; ++ info.dev_attrs = dev_attrs; ++ info.max_sq_frag_cnt = attr->cap.max_send_sge; ++ info.max_rq_frag_cnt = attr->cap.max_recv_sge; ++ info.max_inline_data = attr->cap.max_inline_data; ++ iwuqp->recv_sges = calloc(attr->cap.max_recv_sge, sizeof(struct ibv_sge)); ++ if (!iwuqp->recv_sges) { ++ errno = ENOMEM; ++ goto err_destroy_lock; ++ } ++ iwuqp->sq_sig_all = attr->sq_sig_all; ++ iwuqp->qp_type = attr->qp_type; ++ ++ info.wqe_alloc_db = ++ (__u32 *)((__u8 *)iwvctx->sq_db + ZXDH_DB_SQ_OFFSET); ++ info.abi_ver = iwvctx->abi_ver; ++ info.legacy_mode = iwvctx->legacy_mode; ++ info.sq_wrtrk_array = calloc(info.sqdepth, sizeof(*info.sq_wrtrk_array)); ++ if (info.sq_wrtrk_array == NULL) { ++ errno = ENOMEM; ++ goto err_free_rsges; ++ } ++ ++ info.rq_wrid_array = calloc(info.rqdepth, sizeof(*info.rq_wrid_array)); ++ if (info.rq_wrid_array == NULL) { ++ errno = ENOMEM; ++ goto err_free_sq_wrtrk; ++ } ++ ++ info.type = ZXDH_QP_TYPE_ROCE_RC; ++ status = zxdh_vmapped_qp_with_ext_buff(iwuqp, pd, attr, &info, buf); ++ if (status != 0) { ++ errno = EINVAL; ++ goto err_free_rq_wrid; ++ } ++ ++ iwuqp->qp.back_qp = &iwuqp->vqp.qp; ++ iwuqp->qp.lock = &iwuqp->lock; ++ iwuqp->qp.quanta_lock = &iwuqp->quanta_lock; ++ ++ iwuqp->qp.split_sg_list = calloc(2* dev_attrs->max_hw_read_sges, sizeof(*iwuqp->qp.split_sg_list)); ++ if (iwuqp->qp.split_sg_list == NULL) { ++ errno = ENOMEM; ++ goto err_free_vmapped_qp; ++ } ++ ++ status = zxdh_qp_init(&iwuqp->qp, &info); ++ if (status != 0) { ++ errno = EINVAL; ++ goto err_free_sg_list; ++ } ++ iwuqp->qp.mtu = mtu_enum_to_int(IBV_MTU_1024); ++ iwuqp->qp.ext_buf_mode = iwuqp->ext_buf_mode; ++ attr->cap.max_send_wr = info.sqdepth >> info.sqshift; ++ attr->cap.max_recv_wr = info.rqdepth; ++ ++ attr->cap.max_send_sge = info.max_sq_frag_cnt; ++ attr->cap.max_recv_sge = info.max_rq_frag_cnt; ++ ++ return &iwuqp->vqp.qp; ++ ++err_free_sg_list: ++ free(iwuqp->qp.split_sg_list); ++err_free_vmapped_qp: ++ ibv_cmd_destroy_qp(&iwuqp->vqp.qp); /* free verbs qp if created */ ++err_free_rq_wrid: ++ free(info.rq_wrid_array); ++err_free_sq_wrtrk: ++ free(info.sq_wrtrk_array); ++err_free_rsges: ++ free(iwuqp->recv_sges); ++err_destroy_lock: ++ { int __ret = pthread_spin_destroy(&iwuqp->quanta_lock); (void)__ret; } ++err_destroy_quanta_lock: ++ { int __ret = pthread_spin_destroy(&iwuqp->lock); (void)__ret; } ++err_free_qp: ++ free(iwuqp); ++ return NULL; ++} +diff -ruN baseline/providers/zrdma/zxdh_devx.c.tmp b/providers/zrdma/zxdh_devx.c.tmp +--- a/providers/zrdma/zxdh_devx.c.tmp 1970-01-01 08:00:00.000000000 +0800 ++++ b/providers/zrdma/zxdh_devx.c.tmp 2026-08-06 15:29:37.556696936 +0800 +@@ -0,0 +1,941 @@ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "zxdh_zrdma.h" ++#include "zxdh_abi.h" ++#include "zxdh_devx.h" ++ ++static inline int get_cq_size(int ncqe) ++{ ++ if (ncqe < ZXDH_U_MINCQ_SIZE) ++ ncqe = ZXDH_U_MINCQ_SIZE; ++ return ncqe; ++} ++ ++static inline size_t get_cq_total_bytes(__u32 cq_size) ++{ ++ return roundup(cq_size * ZXDH_CQ_CQE_SIZE, ZXDH_HW_PAGE_SIZE); ++} ++ ++static int zxdh_get_validated_cq_size(struct zxdh_uvcontext *iwvctx, ++ int requested_cqe, __u32 *cq_size) ++{ ++ struct zxdh_dev_attrs *dev_attrs; ++ dev_attrs = &iwvctx->dev_attrs; ++ ++ if (requested_cqe < ZXDH_MIN_CQ_SIZE || ++ requested_cqe > dev_attrs->max_hw_cq_size) { ++ errno = EINVAL; ++ return errno; ++ } ++ ++ *cq_size = get_cq_size(requested_cqe); ++ *cq_size = zxdh_cq_round_up(*cq_size); ++ if (*cq_size > dev_attrs->max_hw_cq_size) { ++ errno = EINVAL; ++ return errno; ++ } ++ ++ return 0; ++} ++ ++static void zxdh_calculate_cq_memory_size(__u32 cq_size, __u32 *cq_pages, ++ size_t *total_size) ++{ ++ size_t cq_bytes; ++ size_t shadow_size; ++ ++ cq_bytes = get_cq_total_bytes(cq_size); ++ *cq_pages = cq_bytes >> ZXDH_HW_PAGE_SHIFT; ++ shadow_size = roundup(ZXDH_DB_SHADOW_AREA_SIZE, ZXDH_HW_PAGE_SIZE); ++ *total_size = (*cq_pages << ZXDH_HW_PAGE_SHIFT) + shadow_size; ++} ++ ++static int zxdh_devx_cq_init(struct zxdh_udevx_cq *cq, ++ struct zxdh_udevx_cq_init_info *info) ++{ ++ cq->devx_cq.verbs_cq.cq.cqe = info->cq_size; ++ cq->cq_base = info->cq_base; ++ cq->cq_id = info->cq_id; ++ cq->cq_size = info->cq_size; ++ cq->shadow_area = info->shadow_area; ++ cq->polarity = 1; ++ cq->devx_cq.cqn = info->cq_id; ++ cq->devx_cq.cqe_num = info->cq_size; ++ cq->devx_cq.polarity = 1; ++ cq->devx_cq.shadow_offset = info->shadow_offset; ++ cq->devx_cq.shadow_size = info->shadow_size; ++ return 0; ++} ++ ++int _zxdh_devx_get_cq_init_size(struct ibv_context *context, ++ struct zxdh_devx_cq_init_attr *attr, ++ struct zxdh_devx_init_cq_resp *resp) ++{ ++ size_t total_size; ++ __u32 cq_pages, cq_size; ++ int ret; ++ struct zxdh_uvcontext *iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ ++ ret = zxdh_get_validated_cq_size(iwvctx, attr->cqe_num, &cq_size); ++ if (ret) ++ return ret; ++ ++ zxdh_calculate_cq_memory_size(cq_size, &cq_pages, &total_size); ++ ++ resp->cqe_num = cq_size; ++ resp->total_size = total_size; ++ resp->shadow_offset = cq_pages << ZXDH_HW_PAGE_SHIFT; ++ return 0; ++} ++ ++static int zxdh_devx_modify_cq_overflow_check_en(struct zxdh_devx_cq *devx_cq, __u32 overflow_check_en) ++{ ++ int ret = 0; ++ ++ if (!devx_cq) { ++ return -EINVAL; ++ } ++ ++ ret = zxdh_devx_modify_cq(&devx_cq->verbs_cq.cq, overflow_check_en); ++ if (ret) ++ return ret; ++ ++ return 0; ++} ++ ++struct zxdh_devx_cq *_zxdh_devx_create_cq(struct ibv_context *context, ++ int cqe_num, ++ struct ibv_comp_channel *channel, ++ int comp_vector, ++ struct zxdh_devx_cq_buffers *buffer ++ ++) ++{ ++ struct zxdh_udevx_cq_init_info info = {}; ++ struct zxdh_ureg_mr reg_mr_cmd = {}; ++ struct zxdh_ureg_mr reg_mr_shadow_cmd = {}; ++ struct zxdh_ucreate_cq cmd = {}; ++ struct zxdh_ucreate_cq_resp resp = {}; ++ struct ib_uverbs_reg_mr_resp reg_mr_resp = {}; ++ struct ib_uverbs_reg_mr_resp reg_mr_shadow_resp = {}; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_udevx_cq *udevx_cq; ++ __u32 cq_pages; ++ size_t total_size; ++ int ret; ++ __u32 overflow_check_en = 0; ++ ++ iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ if (!buffer || !buffer->cq_base) { ++ errno = EINVAL; ++ return NULL; ++ } ++ if (channel || comp_vector) { ++ errno = EINVAL; ++ return NULL; ++ } ++ ret = zxdh_get_validated_cq_size(iwvctx, cqe_num, &info.cq_size); ++ if (ret) { ++ errno = EINVAL; ++ return NULL; ++ } ++ zxdh_calculate_cq_memory_size(info.cq_size, &cq_pages, &total_size); ++ if (buffer->total_size < total_size) { ++ errno = EINVAL; ++ return NULL; ++ } ++ info.cq_base = buffer->cq_base; ++ info.shadow_offset = cq_pages << ZXDH_HW_PAGE_SHIFT; ++ info.shadow_size = ZXDH_DB_SHADOW_AREA_SIZE; ++ ++ udevx_cq = calloc(1, sizeof(*udevx_cq)); ++ if (!udevx_cq) ++ return NULL; ++ ++ udevx_cq->comp_vector = comp_vector; ++ udevx_cq->buf_size = buffer->total_size; ++ reg_mr_cmd.reg_type = ZXDH_MEMREG_TYPE_CQ; ++ reg_mr_cmd.cq_pages = cq_pages; ++ ++ ret = ibv_cmd_reg_mr(&iwvctx->iwupd->ibv_pd, info.cq_base, ++ cq_pages << ZXDH_HW_PAGE_SHIFT, ++ (uintptr_t)info.cq_base, IBV_ACCESS_LOCAL_WRITE, ++ &udevx_cq->vmr, ®_mr_cmd.ibv_cmd, ++ sizeof(reg_mr_cmd), ®_mr_resp, ++ sizeof(reg_mr_resp)); ++ if (ret) { ++ errno = ret; ++ goto err_dereg_mr; ++ } ++ ++ udevx_cq->vmr.ibv_mr.pd = &iwvctx->iwupd->ibv_pd; ++ udevx_cq->devx_cq.verbs_cq.cq.context = context; ++ ++ info.shadow_area = (void *)(info.cq_base + info.shadow_offset); ++ ++ reg_mr_shadow_cmd.reg_type = ZXDH_MEMREG_TYPE_CQ; ++ reg_mr_shadow_cmd.cq_pages = 1; ++ ++ ret = ibv_cmd_reg_mr(&iwvctx->iwupd->ibv_pd, info.shadow_area, ++ ZXDH_HW_PAGE_SIZE, (uintptr_t)info.shadow_area, ++ IBV_ACCESS_LOCAL_WRITE, &udevx_cq->vmr_shadow_area, ++ ®_mr_shadow_cmd.ibv_cmd, ++ sizeof(reg_mr_shadow_cmd), ®_mr_shadow_resp, ++ sizeof(reg_mr_shadow_resp)); ++ if (ret) { ++ errno = ret; ++ goto err_dereg_shadow; ++ } ++ ++ udevx_cq->vmr_shadow_area.ibv_mr.pd = &iwvctx->iwupd->ibv_pd; ++ ++ cmd.user_cq_buf = (__u64)((uintptr_t)info.cq_base); ++ cmd.user_shadow_area = (__u64)((uintptr_t)info.shadow_area); ++ ++ ret = ibv_cmd_create_cq(context, info.cq_size, channel, comp_vector, ++ &udevx_cq->devx_cq.verbs_cq.cq, &cmd.ibv_cmd, ++ sizeof(cmd), &resp.ibv_resp, sizeof(resp)); ++ if (ret) { ++ errno = ret; ++ goto err_dereg_shadow; ++ } ++ info.cq_id = resp.cq_id; ++ ++ ret = zxdh_devx_modify_cq_overflow_check_en(&udevx_cq->devx_cq, overflow_check_en); ++ if (ret) { ++ errno = ret; ++ goto err_dereg_shadow; ++ } ++ ++ zxdh_devx_cq_init(udevx_cq, &info); ++ return &udevx_cq->devx_cq; ++ ++err_dereg_shadow: ++ ibv_cmd_dereg_mr(&udevx_cq->vmr); ++err_dereg_mr: ++ free(udevx_cq); ++ return NULL; ++} ++ ++int _zxdh_devx_destroy_cq(struct zxdh_devx_cq *cq) ++{ ++ struct zxdh_udevx_cq *udevx_cq; ++ int ret; ++ ++ if (!cq) { ++ return -EINVAL; ++ } ++ ++ udevx_cq = container_of(cq, struct zxdh_udevx_cq, devx_cq); ++ ++ ret = ibv_cmd_destroy_cq(&cq->verbs_cq.cq); ++ if (ret) ++ goto err; ++ ++ ibv_cmd_dereg_mr(&udevx_cq->vmr); ++ ++ free(udevx_cq); ++ return 0; ++ ++err: ++ return ret; ++} ++ ++int _zxdh_devx_get_qp_init_size(struct ibv_context *ibv_ctx, ++ struct zxdh_devx_qp_init_attr *attr, ++ struct zxdh_devx_init_qp_resp *resp) ++{ ++ int status; ++ size_t sqsize, rqsize, shadow_size, totalqpsize; ++ __u32 sqdepth, rqdepth; ++ __u8 sqshift, rqshift; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ ++ if (!ibv_ctx || !attr || !resp) { ++ return -EINVAL; ++ } ++ ++ if (attr->qp_type != IBV_QPT_RC) { ++ return -EOPNOTSUPP; ++ } ++ ++ iwvctx = container_of(ibv_ctx, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ ++ if (attr->cap.max_send_sge > dev_attrs->max_hw_wq_frags || ++ attr->cap.max_recv_sge > dev_attrs->max_hw_wq_frags) { ++ return -EINVAL; ++ } ++ ++ if (attr->cap.max_inline_data > dev_attrs->max_hw_inline) { ++ zxdh_dbg(ZXDH_DBG_QP, "max_inline_data over max_hw_inline\n"); ++ attr->cap.max_inline_data = dev_attrs->max_hw_inline; ++ } ++ ++ zxdh_get_sq_wqe_shift(attr->cap.max_send_sge, attr->cap.max_inline_data, ++ &sqshift); ++ status = zxdh_devx_get_sqdepth(dev_attrs, attr->cap.max_send_wr, sqshift, ++ &sqdepth); ++ if (status) { ++ return -EINVAL; ++ } ++ ++ zxdh_get_rq_wqe_shift(attr->cap.max_recv_sge, &rqshift); ++ status = zxdh_devx_get_rqdepth(dev_attrs, attr->cap.max_recv_wr, rqshift, ++ &rqdepth); ++ if (status) { ++ return -EINVAL; ++ } ++ ++ sqsize = roundup(sqdepth * ZXDH_QP_SQE_MIN_SIZE, ZXDH_HW_PAGE_SIZE); ++ rqsize = roundup((rqdepth << rqshift) * ZXDH_QP_RQE_MIN_SIZE, ZXDH_HW_PAGE_SIZE); ++ shadow_size = roundup(ZXDH_DB_SHADOW_AREA_SIZE, ZXDH_HW_PAGE_SIZE); ++ totalqpsize = rqsize + sqsize + shadow_size; ++ ++ resp->total_size = totalqpsize; ++ resp->rq_offset = sqsize; ++ resp->shadow_offset = sqsize + rqsize; ++ return 0; ++} ++ ++static int zxdh_devx_vmapped_qp(struct zxdh_udevx_qp *devx_uqp, ++ struct ibv_pd *pd, ++ struct ibv_qp_init_attr *attr, ++ struct zxdh_qp_init_info *info, ++ struct zxdh_devx_qp_buffers *buffer) ++{ ++ struct zxdh_ucreate_qp cmd = {}; ++ struct zxdh_ucreate_qp_resp resp = {}; ++ struct zxdh_ureg_mr reg_mr_cmd = {}; ++ struct ib_uverbs_reg_mr_resp reg_mr_resp = {}; ++ int status; ++ size_t sqsize = 0, rqsize = 0, shadow_size; ++ devx_uqp->buf_size = buffer->total_size; ++ ++ sqsize = roundup(info->sqdepth * ZXDH_QP_SQE_MIN_SIZE, ++ ZXDH_HW_PAGE_SIZE); ++ if (info->rqdepth > 0) { ++ rqsize = roundup((info->rqdepth << info->rqshift) * ZXDH_QP_RQE_MIN_SIZE, ++ ZXDH_HW_PAGE_SIZE); ++ reg_mr_cmd.rq_pages = rqsize >> ZXDH_HW_PAGE_SHIFT; ++ } ++ shadow_size = roundup(ZXDH_DB_SHADOW_AREA_SIZE, ZXDH_HW_PAGE_SIZE); ++ if ((sqsize + rqsize + shadow_size) != buffer->total_size) { ++ return -EINVAL; ++ } ++ ++ reg_mr_cmd.reg_type = ZXDH_MEMREG_TYPE_QP; ++ reg_mr_cmd.sq_pages = sqsize >> ZXDH_HW_PAGE_SHIFT; ++ status = ibv_cmd_reg_mr(pd, buffer->qp_base, buffer->total_size, ++ (uintptr_t)buffer->qp_base, ++ IBV_ACCESS_LOCAL_WRITE, &devx_uqp->vmr, ++ ®_mr_cmd.ibv_cmd, sizeof(reg_mr_cmd), ++ ®_mr_resp, sizeof(reg_mr_resp)); ++ if (status) ++ return status; ++ ++ cmd.user_wqe_bufs = (__u64)((uintptr_t)buffer->qp_base); ++ cmd.user_compl_ctx = (__u64)(uintptr_t)&devx_uqp->devx_qp; ++ ++ status = ibv_cmd_create_qp(pd, &devx_uqp->devx_qp.verbs_qp.qp, attr, ++ &cmd.ibv_cmd, sizeof(cmd), &resp.ibv_resp, ++ sizeof(struct zxdh_ucreate_qp_resp)); ++ if (status) { ++ goto err_qp; ++ } ++ ++ info->sq_size = resp.actual_sq_size; ++ info->rq_size = resp.actual_rq_size; ++ info->qp_caps = resp.qp_caps; ++ info->qp_id = resp.qp_id; ++ devx_uqp->zxdh_drv_opt = resp.zxdh_drv_opt; ++ devx_uqp->devx_qp.verbs_qp.qp.qp_num = resp.qp_id; ++ ++ if (!attr->send_cq || !attr->recv_cq) { ++ status = -EINVAL; ++ goto err_qp; ++ } ++ ++ devx_uqp->send_cq = ++ container_of(attr->send_cq, struct zxdh_udevx_cq, devx_cq.verbs_cq.cq); ++ devx_uqp->recv_cq = ++ container_of(attr->recv_cq, struct zxdh_udevx_cq, devx_cq.verbs_cq.cq); ++ if (devx_uqp->send_cq) { ++ devx_uqp->send_cq->uqp = devx_uqp; ++ } ++ if (devx_uqp->recv_cq) { ++ devx_uqp->recv_cq->uqp = devx_uqp; ++ } ++ ++ return 0; ++err_qp: ++ ibv_cmd_dereg_mr(&devx_uqp->vmr); ++ return status; ++} ++ ++enum zxdh_status_code zxdh_devx_qp_init(struct zxdh_udevx_qp *qp, ++ struct zxdh_qp_init_info *info) ++{ ++ enum zxdh_status_code ret_code = 0; ++ qp->qp_caps = info->qp_caps; ++ qp->qp_id = info->qp_id; ++ qp->sq_size = info->sq_size; ++ qp->rq_size = info->rq_size; ++ qp->qp_type = info->type; ++ qp->devx_qp.wqe_alloc_db = info->wqe_alloc_db; ++ qp->devx_qp.qpn = info->qp_id; ++ qp->devx_qp.sqdepth = info->sqdepth; ++ qp->devx_qp.sqshift = info->sqshift; ++ qp->devx_qp.rqdepth = info->rqdepth; ++ qp->devx_qp.rqshift = info->rqshift; ++ qp->devx_qp.sq_polarity = 1; ++ qp->devx_qp.rq_polarity = 0; ++ qp->devx_qp.max_sq_frag_cnt = info->max_sq_frag_cnt; ++ qp->devx_qp.max_rq_frag_cnt = info->max_rq_frag_cnt; ++ qp->devx_qp.max_inline_data = (info->max_inline_data == 0) ? ++ ZXDH_MAX_INLINE_DATA_SIZE : ++ info->max_inline_data; ++ qp->devx_qp.rq_wqe_size_multiplier = 1 << info->rqshift; ++ qp->devx_qp.shadow_init_value = 0x8000; ++ qp->devx_qp.shadow_size = ZXDH_DB_SHADOW_AREA_SIZE; ++ return ret_code; ++} ++ ++struct zxdh_devx_qp *_zxdh_devx_create_qp(struct ibv_pd *pd, ++ struct ibv_qp_init_attr *attr, ++ struct zxdh_devx_qp_buffers *buffer) ++{ ++ struct zxdh_qp_init_info info = { 0 }; ++ struct zxdh_uvcontext *iwvctx; ++ int status; ++ struct zxdh_dev_attrs *dev_attrs; ++ struct zxdh_udevx_qp *devx_uqp; ++ struct ibv_context *ibv_ctx; ++ ++ if (!pd || !attr || !buffer) { ++ errno = EINVAL; ++ return NULL; ++ } ++ ++ if (buffer->qp_base == NULL || buffer->total_size == 0) { ++ errno = EINVAL; ++ return NULL; ++ } ++ if (attr->qp_type != IBV_QPT_RC) { ++ errno = EOPNOTSUPP; ++ return NULL; ++ } ++ ibv_ctx = pd->context; ++ iwvctx = container_of(ibv_ctx, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ if (attr->cap.max_send_sge > dev_attrs->max_hw_wq_frags || ++ attr->cap.max_recv_sge > dev_attrs->max_hw_wq_frags) { ++ errno = EINVAL; ++ return NULL; ++ } ++ ++ if (attr->cap.max_inline_data > dev_attrs->max_hw_inline) { ++ zxdh_dbg(ZXDH_DBG_QP, "max_inline_data over max_hw_inline\n"); ++ attr->cap.max_inline_data = dev_attrs->max_hw_inline; ++ } ++ ++ zxdh_get_sq_wqe_shift(attr->cap.max_send_sge, attr->cap.max_inline_data, ++ &info.sqshift); ++ status = zxdh_devx_get_sqdepth(dev_attrs, attr->cap.max_send_wr, ++ info.sqshift, &info.sqdepth); ++ if (status) { ++ errno = EINVAL; ++ return NULL; ++ } ++ ++ zxdh_get_rq_wqe_shift(attr->cap.max_recv_sge, &info.rqshift); ++ status = zxdh_devx_get_sqdepth(dev_attrs, attr->cap.max_recv_wr, ++ info.rqshift, &info.rqdepth); ++ if (status) { ++ errno = EINVAL; ++ return NULL; ++ } ++ ++ devx_uqp = memalign(1024, sizeof(*devx_uqp)); ++ if (!devx_uqp) { ++ errno = ENOMEM; ++ return NULL; ++ } ++ memset(devx_uqp, 0, sizeof(*devx_uqp)); ++ ++ devx_uqp->devx_qp.sq_sig_all = attr->sq_sig_all; ++ devx_uqp->devx_qp.verbs_qp.qp.qp_type = attr->qp_type; ++ devx_uqp->devx_qp.verbs_qp.qp.context = ibv_ctx; ++ info.sq_size = info.sqdepth >> info.sqshift; ++ info.rq_size = info.rqdepth; ++ attr->cap.max_send_wr = info.sq_size; ++ attr->cap.max_recv_wr = info.rq_size; ++ ++ info.dev_attrs = dev_attrs; ++ info.max_sq_frag_cnt = attr->cap.max_send_sge; ++ info.max_rq_frag_cnt = attr->cap.max_recv_sge; ++ ++ info.wqe_alloc_db = ++ (__u32 *)((__u8 *)iwvctx->sq_db + ZXDH_DB_SQ_OFFSET); ++ info.abi_ver = iwvctx->abi_ver; ++ info.legacy_mode = iwvctx->legacy_mode; ++ ++ info.type = ZXDH_QP_TYPE_ROCE_RC; ++ status = zxdh_devx_vmapped_qp(devx_uqp, pd, attr, &info, buffer); ++ if (status) { ++ errno = status; ++ goto err_free_qp; ++ } ++ ++ info.max_sq_frag_cnt = attr->cap.max_send_sge; ++ info.max_rq_frag_cnt = attr->cap.max_recv_sge; ++ info.max_inline_data = attr->cap.max_inline_data; ++ ++ zxdh_devx_qp_init(devx_uqp, &info); ++ attr->cap.max_send_wr = (info.sqdepth - ZXDH_SQ_RSVD) >> info.sqshift; ++ attr->cap.max_recv_wr = info.rqdepth - ZXDH_RQ_RSVD; ++ ++ return &devx_uqp->devx_qp; ++ ++err_free_qp: ++ free(devx_uqp); ++ return NULL; ++} ++ ++static int zxdh_destroy_devx_vmapped_qp(struct zxdh_udevx_qp *devx_uqp) ++{ ++ int ret; ++ ++ ret = ibv_cmd_destroy_qp(&devx_uqp->devx_qp.verbs_qp.qp); ++ if (ret) ++ return ret; ++ ++ ibv_cmd_dereg_mr(&devx_uqp->vmr); ++ ++ return 0; ++} ++ ++int _zxdh_devx_destroy_qp(struct zxdh_devx_qp *devx_qp) ++{ ++ int ret; ++ struct zxdh_udevx_qp *devx_uqp; ++ ++ if (!devx_qp) { ++ return -EINVAL; ++ } ++ ++ devx_uqp = container_of(devx_qp, struct zxdh_udevx_qp, devx_qp); ++ ++ devx_uqp->destroy_pending = true; ++ ++ ret = zxdh_destroy_devx_vmapped_qp(devx_uqp); ++ if (ret) ++ goto err; ++ free(devx_uqp); ++ ++ return 0; ++ ++err: ++ return ret; ++} ++ ++static void zxdh_init_qp_indices(struct zxdh_udevx_qp *devx_uqp) ++{ ++ devx_uqp->devx_qp.sq_polarity = 0; ++ devx_uqp->devx_qp.rq_polarity = 0; ++ devx_uqp->devx_qp.rq_signature = 0; ++} ++ ++int _zxdh_devx_modify_qp(struct zxdh_devx_qp *devx_qp, struct ibv_qp_attr *attr, ++ int attr_mask) ++{ ++ struct zxdh_udevx_qp *devx_uqp; ++ struct zxdh_umodify_qp_resp resp = {}; ++ struct ibv_modify_qp cmd = {}; ++ struct zxdh_umodify_qp cmd_ex = {}; ++ int ret = 0; ++ __u16 mtu = 0; ++ ++ if (!devx_qp || !attr) { ++ return -EINVAL; ++ } ++ ++ devx_uqp = container_of(devx_qp, struct zxdh_udevx_qp, devx_qp); ++ if (!ret && (attr_mask & IBV_QP_PATH_MTU) && ++ devx_qp->verbs_qp.qp.qp_type == IBV_QPT_RC) { ++ mtu = mtu_enum_to_int(attr->path_mtu); ++ if (mtu == 0) ++ return -EINVAL; ++ } ++ if (attr_mask & IBV_QP_STATE || attr_mask & IBV_QP_RATE_LIMIT) { ++ ret = ibv_cmd_modify_qp_ex(&devx_qp->verbs_qp.qp, attr, ++ attr_mask, &cmd_ex.ibv_cmd, ++ sizeof(cmd_ex), &resp.ibv_resp, ++ sizeof(resp)); ++ } else { ++ ret = ibv_cmd_modify_qp(&devx_qp->verbs_qp.qp, attr, attr_mask, ++ &cmd, sizeof(cmd)); ++ } ++ ++ if (!ret && (attr_mask & IBV_QP_STATE) && ++ attr->qp_state == IBV_QPS_RESET) { ++ zxdh_init_qp_indices(devx_uqp); ++ } ++ ++ return ret; ++} ++ ++struct ibv_mr *_zxdh_devx_reg_mr(struct ibv_pd *pd, void *host_addr, size_t length, uint64_t dev_addr, unsigned int access) ++{ ++ struct zxdh_umr *umr; ++ struct zxdh_ureg_mr cmd; ++ struct zxdh_ureg_mr_resp resp = {}; ++ int err; ++ ++ umr = malloc(sizeof(*umr)); ++ if (!umr) ++ return NULL; ++ ++ cmd.reg_type = ZXDH_MEMREG_TYPE_EXTERNEL_MEM; ++ err = ibv_cmd_reg_mr(pd, host_addr, length, dev_addr, access, &umr->vmr, ++ &cmd.ibv_cmd, sizeof(cmd), &resp.ibv_resp, ++ sizeof(resp)); ++ if (err) { ++ free(umr); ++ errno = err; ++ return NULL; ++ } ++ ++ umr->acc_flags = access; ++ umr->host_page_size = resp.host_page_size; ++ return &umr->vmr.ibv_mr; ++} ++ ++int _zxdh_devx_dereg_mr(struct ibv_mr *mr) ++{ ++ struct verbs_mr *vmr = verbs_get_mr(mr); ++ int ret; ++ ++ ret = ibv_cmd_dereg_mr(vmr); ++ if (ret) ++ return ret; ++ ++ free(vmr); ++ return 0; ++} ++ ++static struct ibv_cq *ucreate_cq_with_ext_buf(struct ibv_context *context, ++ struct ibv_cq_init_attr_ex *attr_ex, ++ struct zxdh_ext_buf *buffer) ++{ ++ struct zxdh_cq_init_info info = {}; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_ucq *iwucq; ++ struct zxdh_ucreate_cq_ex cmd = {}; ++ struct zxdh_ucreate_cq_ex_resp resp = {}; ++ size_t total_size; ++ __u32 cq_pages, shadow_offset; ++ int ret, ncqe; ++ int lock_inited = 0; ++ ++ iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ ret = zxdh_get_validated_cq_size(iwvctx, attr_ex->cqe, &info.cq_size); ++ if (ret) { ++ errno = EINVAL; ++ return NULL; ++ } ++ ++ iwucq = calloc(1, sizeof(*iwucq)); ++ if (!iwucq) { ++ errno = ENOMEM; ++ return NULL; ++ } ++ memset(iwucq, 0, sizeof(*iwucq)); ++ ++ ret = pthread_spin_init(&iwucq->lock, PTHREAD_PROCESS_PRIVATE); ++ if (ret) { ++ errno = (ret > 0) ? ret : -ret; ++ goto err; ++ } ++ lock_inited = 1; ++ ++ iwucq->resize_enable = false; ++ iwucq->is_ext_buf_mode = true; ++ iwucq->comp_vector = attr_ex->comp_vector; ++ ++ zxdh_calculate_cq_memory_size(info.cq_size, &cq_pages, &total_size); ++ buffer->total_size = total_size; ++ iwucq->buf_size = total_size; ++ shadow_offset = cq_pages << ZXDH_HW_PAGE_SHIFT; ++ iwucq->vmr.ibv_mr.pd = &iwvctx->iwupd->ibv_pd; ++ info.cq_base = buffer->host_addr; ++ info.shadow_area = (__le64 *)((__u8 *)info.cq_base + shadow_offset); ++ ++ ncqe = attr_ex->cqe; ++ attr_ex->cqe = info.cq_size; ++ ++ cmd.user_cq_buf = (__u64)((uintptr_t)info.cq_base); ++ cmd.user_shadow_area = (__u64)((uintptr_t)info.shadow_area); ++ cmd.dev_cq_buf = buffer->dev_addr; ++ cmd.dev_shadow_area = buffer->dev_addr + shadow_offset; ++ ret = ibv_cmd_create_cq_ex_wrap(context, attr_ex, &iwucq->verbs_cq, &cmd.ibv_cmd, sizeof(cmd), &resp.ibv_resp, sizeof(resp)); ++ if (ret) { ++ errno = (ret > 0) ? ret : -ret; ++ goto err; ++ } ++ info.cq_id = resp.cq_id; ++ iwucq->verbs_cq.cq.cqe = ncqe; ++ info.cqe_alloc_db = (__u32*)((__u8*)iwvctx->cq_db + ZXDH_DB_CQ_OFFSET); ++ ++ zxdh_cq_init(&iwucq->cq, &info); ++ return &iwucq->verbs_cq.cq; ++err: ++ if (lock_inited) { ++ int __destroy_ret = pthread_spin_destroy(&iwucq->lock); ++ (void)__destroy_ret; ++ } ++ free(iwucq); ++ return NULL; ++} ++ ++struct ibv_cq *_zxdh_devx_create_cq_with_ext_buf(struct ibv_context *context, ++ int cqe, struct ibv_comp_channel *channel, ++ int comp_vector, struct zxdh_ext_buf *buf) ++{ ++ struct ibv_cq *ibv_cq; ++ struct ibv_cq_init_attr_ex attr_ex = { .cqe = cqe, .channel = channel, .comp_vector = comp_vector }; ++ ++ if (!buf || !buf->host_addr || !buf->dev_addr) { ++ errno = EINVAL; ++ return NULL; ++ } ++ ++ if (channel || comp_vector) { ++ errno = EINVAL; ++ return NULL; ++ } ++ ++ ibv_cq = ucreate_cq_with_ext_buf(context, &attr_ex, buf); ++ if (!ibv_cq) { ++ return NULL; ++ } ++ ++ return ibv_cq; ++} ++ ++static int zxdh_vmapped_qp_with_ext_buff(struct zxdh_uqp *iwuqp, ++ struct ibv_pd *pd, ++ struct ibv_qp_init_attr *attr, ++ struct zxdh_qp_init_info *info, ++ struct zxdh_ext_buf *buffer) ++{ ++ struct zxdh_ucreate_qp cmd = {}; ++ struct zxdh_ucreate_qp_resp resp = {}; ++ int ret; ++ size_t sqsize, rqsize, shadow_size; ++ ++ if (!attr->send_cq || !attr->recv_cq) { ++ return -EINVAL; ++ } ++ ++ sqsize = roundup(info->sqdepth * ZXDH_QP_SQE_MIN_SIZE, ZXDH_HW_PAGE_SIZE); ++ rqsize = roundup((info->rqdepth << info->rqshift) * ZXDH_QP_RQE_MIN_SIZE, ZXDH_HW_PAGE_SIZE); ++ shadow_size = roundup(ZXDH_DB_SHADOW_AREA_SIZE, ZXDH_HW_PAGE_SIZE); ++ buffer->total_size = sqsize + rqsize + shadow_size; ++ iwuqp->buf_size = buffer->total_size; ++ ++ info->sq = (struct zxdh_qp_sq_quanta *)buffer->host_addr; ++ info->rq = (struct zxdh_qp_rq_quanta *)((__u8 *)buffer->host_addr + sqsize); ++ info->shadow_area = (__le64 *)((__u8 *)buffer->host_addr + sqsize + rqsize); ++ ++ cmd.user_wqe_bufs = (__u64)((uintptr_t)info->sq); ++ cmd.user_compl_ctx = (__u64)(uintptr_t)&iwuqp->qp; ++ cmd.dev_wqe_bufs = buffer->dev_addr; ++ cmd.rq_offset = (__u32)sqsize; ++ cmd.shadow_offset = (__u32)(sqsize + rqsize); ++ ++ ret = ibv_cmd_create_qp(pd, &iwuqp->vqp.qp, attr, ++ &cmd.ibv_cmd, sizeof(cmd), &resp.ibv_resp, ++ sizeof(struct zxdh_ucreate_qp_resp)); ++ if (ret != 0) { ++ return ret; ++ } ++ ++ info->sq_size = resp.actual_sq_size; ++ info->rq_size = resp.actual_rq_size; ++ info->qp_caps = resp.qp_caps; ++ info->qp_id = resp.qp_id; ++ iwuqp->zxdh_drv_opt = resp.zxdh_drv_opt; ++ iwuqp->qp.qp_id = resp.qp_id; ++ iwuqp->vqp.qp.qp_num = resp.qp_id; ++ iwuqp->send_cq = container_of(attr->send_cq, struct zxdh_ucq, verbs_cq.cq); ++ iwuqp->recv_cq = container_of(attr->recv_cq, struct zxdh_ucq, verbs_cq.cq); ++ iwuqp->send_cq->uqp = iwuqp; ++ iwuqp->recv_cq->uqp = iwuqp; ++ ++ return 0; ++} ++ ++struct ibv_qp *_zxdh_devx_create_qp_with_ext_buf(struct ibv_pd *pd, ++ struct ibv_qp_init_attr *attr, ++ struct zxdh_ext_buf *buf) ++{ ++ struct zxdh_uqp *iwuqp; ++ struct zxdh_qp_init_info info = {}; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ int status; ++ ++ if (!pd || !attr || !buf || !buf->host_addr || !buf->dev_addr) { ++ errno = EINVAL; ++ return NULL; ++ } ++ ++ if (attr->qp_type != IBV_QPT_RC || attr->srq != NULL) { ++ errno = EOPNOTSUPP; ++ return NULL; ++ } ++ ++ iwvctx = container_of(pd->context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ ++ if (attr->cap.max_send_sge > dev_attrs->max_hw_wq_frags || ++ attr->cap.max_recv_sge > dev_attrs->max_hw_wq_frags) { ++ errno = EINVAL; ++ return NULL; ++ } ++ ++ if (attr->cap.max_inline_data > dev_attrs->max_hw_inline) { ++ zxdh_dbg(ZXDH_DBG_QP, "max_inline_data over max_hw_inline\n"); ++ attr->cap.max_inline_data = dev_attrs->max_hw_inline; ++ } ++ ++ zxdh_get_sq_wqe_shift(attr->cap.max_send_sge, attr->cap.max_inline_data, ++ &info.sqshift); ++ status = zxdh_get_sqdepth(dev_attrs, attr->cap.max_send_wr, ++ info.sqshift, &info.sqdepth); ++ if (status) { ++ errno = EINVAL; ++ return NULL; ++ } ++ ++ zxdh_get_rq_wqe_shift(attr->cap.max_recv_sge, &info.rqshift); ++ status = zxdh_get_rqdepth(dev_attrs, attr->cap.max_recv_wr, ++ info.rqshift, &info.rqdepth); ++ if (status) { ++ errno = EINVAL; ++ return NULL; ++ } ++ ++ iwuqp = memalign(1024, sizeof(*iwuqp)); ++ if (!iwuqp) { ++ errno = ENOMEM; ++ return NULL; ++ } ++ memset(iwuqp, 0, sizeof(*iwuqp)); ++ iwuqp->ext_buf_mode = 1; ++ if (pthread_spin_init(&iwuqp->lock, PTHREAD_PROCESS_PRIVATE)) { ++ errno = ENOMEM; ++ goto err_free_qp; ++ } ++ if (pthread_spin_init(&iwuqp->quanta_lock, PTHREAD_PROCESS_PRIVATE)) { ++ errno = ENOMEM; ++ goto err_destroy_quanta_lock; ++ } ++ ++ attr->cap.max_send_wr = info.sqdepth >> info.sqshift; ++ attr->cap.max_recv_wr = info.rqdepth; ++ info.dev_attrs = dev_attrs; ++ info.max_sq_frag_cnt = attr->cap.max_send_sge; ++ info.max_rq_frag_cnt = attr->cap.max_recv_sge; ++ info.max_inline_data = attr->cap.max_inline_data; ++ iwuqp->recv_sges = calloc(attr->cap.max_recv_sge, sizeof(struct ibv_sge)); ++ if (!iwuqp->recv_sges) { ++ errno = ENOMEM; ++ goto err_destroy_lock; ++ } ++ iwuqp->sq_sig_all = attr->sq_sig_all; ++ iwuqp->qp_type = attr->qp_type; ++ ++ info.wqe_alloc_db = ++ (__u32 *)((__u8 *)iwvctx->sq_db + ZXDH_DB_SQ_OFFSET); ++ info.abi_ver = iwvctx->abi_ver; ++ info.legacy_mode = iwvctx->legacy_mode; ++ info.sq_wrtrk_array = calloc(info.sqdepth, sizeof(*info.sq_wrtrk_array)); ++ if (info.sq_wrtrk_array == NULL) { ++ errno = ENOMEM; ++ goto err_free_rsges; ++ } ++ ++ info.rq_wrid_array = calloc(info.rqdepth, sizeof(*info.rq_wrid_array)); ++ if (info.rq_wrid_array == NULL) { ++ errno = ENOMEM; ++ goto err_free_sq_wrtrk; ++ } ++ ++ info.type = ZXDH_QP_TYPE_ROCE_RC; ++ status = zxdh_vmapped_qp_with_ext_buff(iwuqp, pd, attr, &info, buf); ++ if (status != 0) { ++ errno = EINVAL; ++ goto err_free_rq_wrid; ++ } ++ ++ iwuqp->qp.back_qp = &iwuqp->vqp.qp; ++ iwuqp->qp.lock = &iwuqp->lock; ++ iwuqp->qp.quanta_lock = &iwuqp->quanta_lock; ++ ++ iwuqp->qp.split_sg_list = calloc(2* dev_attrs->max_hw_read_sges, sizeof(*iwuqp->qp.split_sg_list)); ++ if (iwuqp->qp.split_sg_list == NULL) { ++ errno = ENOMEM; ++ goto err_free_vmapped_qp; ++ } ++ ++ status = zxdh_qp_init(&iwuqp->qp, &info); ++ if (status != 0) { ++ errno = EINVAL; ++ goto err_free_sg_list; ++ } ++ iwuqp->qp.mtu = mtu_enum_to_int(IBV_MTU_1024); ++ iwuqp->qp.ext_buf_mode = iwuqp->ext_buf_mode; ++ attr->cap.max_send_wr = info.sqdepth >> info.sqshift; ++ attr->cap.max_recv_wr = info.rqdepth; ++ ++ attr->cap.max_send_sge = info.max_sq_frag_cnt; ++ attr->cap.max_recv_sge = info.max_rq_frag_cnt; ++ ++ return &iwuqp->vqp.qp; ++ ++err_free_sg_list: ++ free(iwuqp->qp.split_sg_list); ++err_free_vmapped_qp: ++ ibv_cmd_destroy_qp(&iwuqp->vqp.qp); /* free verbs qp if created */ ++err_free_rq_wrid: ++ free(info.rq_wrid_array); ++err_free_sq_wrtrk: ++ free(info.sq_wrtrk_array); ++err_free_rsges: ++ free(iwuqp->recv_sges); ++err_destroy_lock: ++ { int __ret = pthread_spin_destroy(&iwuqp->quanta_lock); (void)__ret; } ++err_destroy_quanta_lock: ++ { int __ret = pthread_spin_destroy(&iwuqp->lock); (void)__ret; } ++err_free_qp: ++ free(iwuqp); ++ return NULL; ++} +diff -ruN baseline/providers/zrdma/zxdh_devx.h b/providers/zrdma/zxdh_devx.h +--- a/providers/zrdma/zxdh_devx.h 1970-01-01 08:00:00.000000000 +0800 ++++ b/providers/zrdma/zxdh_devx.h 2026-08-06 15:29:37.905720905 +0800 +@@ -0,0 +1,74 @@ ++#ifndef ZXDH_RDMA_DEVX_H ++#define ZXDH_RDMA_DEVX_H ++ ++#include "zxdh_dv.h" ++ ++struct zxdh_udevx_qp { ++ struct zxdh_devx_qp devx_qp; ++ struct zxdh_udevx_cq *send_cq; ++ struct zxdh_udevx_cq *recv_cq; ++ struct verbs_mr vmr; ++ __u32 buf_size; ++ __u32 zxdh_drv_opt; ++ __u32 qp_id; ++ __u32 qp_caps; ++ __u32 sq_size; ++ __u32 rq_size; ++ __u8 qp_type; ++ __u8 destroy_pending : 1; ++}; ++ ++struct zxdh_udevx_cq { ++ struct zxdh_devx_cq devx_cq; ++ struct zxdh_udevx_qp *uqp; ++ struct verbs_mr vmr; ++ struct verbs_mr vmr_shadow_area; ++ size_t buf_size; ++ int comp_vector; ++ void *cq_base; ++ void *shadow_area; ++ __u32 cq_id; ++ __u32 cq_size; ++ __u8 polarity; ++}; ++ ++struct zxdh_udevx_cq_init_info { ++ void *cq_base; ++ void *shadow_area; ++ __u32 cq_size; ++ __u32 cq_id; ++ __u32 shadow_offset; ++ __u32 shadow_size; ++}; ++ ++int _zxdh_devx_get_qp_init_size(struct ibv_context *ibv_ctx, ++ struct zxdh_devx_qp_init_attr *attr, ++ struct zxdh_devx_init_qp_resp *resp); ++int _zxdh_devx_destroy_qp(struct zxdh_devx_qp *devx_qp); ++int _zxdh_devx_modify_qp(struct zxdh_devx_qp *devx_qp, struct ibv_qp_attr *attr, ++ int attr_mask); ++struct zxdh_devx_qp *_zxdh_devx_create_qp(struct ibv_pd *pd, ++ struct ibv_qp_init_attr *attr, ++ struct zxdh_devx_qp_buffers *buffer); ++enum zxdh_status_code zxdh_devx_qp_init(struct zxdh_udevx_qp *qp, ++ struct zxdh_qp_init_info *info); ++int _zxdh_devx_get_cq_init_size(struct ibv_context *context, ++ struct zxdh_devx_cq_init_attr *attr, ++ struct zxdh_devx_init_cq_resp *resp); ++struct zxdh_devx_cq *_zxdh_devx_create_cq(struct ibv_context *context, ++ int cqe_num, ++ struct ibv_comp_channel *channel, ++ int comp_vector, ++ struct zxdh_devx_cq_buffers *buffer); ++int _zxdh_devx_destroy_cq(struct zxdh_devx_cq *cq); ++struct ibv_mr *_zxdh_devx_reg_mr(struct ibv_pd *pd, void *host_addr, size_t length, uint64_t dev_addr, unsigned int access); ++int _zxdh_devx_dereg_mr(struct ibv_mr *mr); ++struct ibv_cq *_zxdh_devx_create_cq_with_ext_buf(struct ibv_context *context, ++ int cqe_num, ++ struct ibv_comp_channel *channel, ++ int comp_vector, ++ struct zxdh_ext_buf *buf); ++struct ibv_qp *_zxdh_devx_create_qp_with_ext_buf(struct ibv_pd *pd, ++ struct ibv_qp_init_attr *attr, ++ struct zxdh_ext_buf *buf); ++#endif +diff -ruN baseline/providers/zrdma/zxdh_dv.h b/providers/zrdma/zxdh_dv.h +--- a/providers/zrdma/zxdh_dv.h 2026-08-06 15:37:55.351862693 +0800 ++++ b/providers/zrdma/zxdh_dv.h 2026-08-06 15:29:37.071663626 +0800 +@@ -21,6 +21,10 @@ + extern "C" { + #endif + ++#ifndef ZXDH_HAVE_QP_EX ++#define ZXDH_HAVE_QP_EX 1 ++#endif ++ + enum switch_status { + SWITCH_CLOSE = 0, + SWITCH_OPEN = 1, +@@ -57,17 +61,540 @@ + uint8_t retry_count; + }; + ++struct zxdh_rdma_cap_pa { ++ __u64 cap_pa_node0; ++ __u64 cap_pa_node1; ++}; ++ ++#define CAP_NODE_NUM 2 ++#define NODE1 1 ++#define NODE0 0 ++#define EN_32bit_GROUP_NUM 16 ++#define BIT_O_31 0 ++#define BIT_32_63 1 ++#define BIT_64_95 2 ++#define BIT_96_127 3 ++#define BIT_128_159 4 ++#define BIT_160_191 5 ++#define BIT_192_223 6 ++#define BIT_224_255 7 ++#define BIT_256_287 8 ++#define BIT_288_319 9 ++#define BIT_320_351 10 ++#define BIT_352_383 11 ++#define BIT_384_415 12 ++#define BIT_416_447 13 ++#define BIT_448_479 14 ++#define BIT_480_511 15 ++#define CAP_TX 1 ++#define CAP_RX 2 ++#define FREE_TYPE_NONE 0 ++#define FREE_TYPE_MP 1 ++#define FREE_TYPE_TX 2 ++#define FREE_TYPE_RX 3 ++#define FREE_TYPE_IOVA 4 ++#define FREE_TYPE_HW_OBJ_DATA 5 ++ ++struct zxdh_cap_cfg { ++ uint8_t cap_position; ++ uint64_t size; ++ uint32_t channel_select[CAP_NODE_NUM]; ++ uint32_t channel_open[CAP_NODE_NUM]; ++ uint32_t node_choose[CAP_NODE_NUM]; ++ uint32_t node_select[CAP_NODE_NUM]; ++ uint32_t compare_bit_en[EN_32bit_GROUP_NUM][CAP_NODE_NUM]; ++ uint32_t compare_data[EN_32bit_GROUP_NUM][CAP_NODE_NUM]; ++ uint32_t rdma_time_wrl2d[CAP_NODE_NUM]; ++ uint32_t extra[CAP_NODE_NUM][EN_32bit_GROUP_NUM]; ++ uint32_t cap_data_start_cap; ++}; ++ ++#define MAX_CAP_QPS 4 ++ ++struct zxdh_mp_cap_cfg { ++ bool cap_use_l2d; ++ uint32_t qpn[MAX_CAP_QPS]; ++ uint8_t qpn_num; ++}; ++ ++struct zxdh_mp_cap_gqp_cfg { ++ bool cap_use_l2d; ++ uint16_t gqpid[MAX_CAP_QPS]; ++ uint8_t gqp_num; ++ bool is_vm_capture; ++}; ++ ++enum { ++ MCODE_TYPE_DCQCN = 1, ++ MCODE_TYPE_RTT, ++ MCODE_TYPE_BASERTT, ++ MCODE_TYPE_PID, ++ MCODE_TYPE_WUMENG = 6, ++}; ++ ++struct zxdh_mp_cap_gqp { ++ uint8_t mcode_type; ++ uint8_t gqp_num; ++ uint16_t gqpid[MAX_CAP_QPS]; ++ uint64_t cap_pa; ++}; ++ ++struct zxdh_cap_gqp { ++ uint16_t gqpid[MAX_CAP_QPS]; ++ uint8_t gqp_num; ++}; ++ ++#define MAX_ACTIVE_GQP_NUM 16 ++struct zxdh_active_vhca_gqps { ++ uint16_t vhca_id; ++ uint16_t gqp_id[MAX_ACTIVE_GQP_NUM]; ++ uint8_t gqp_num; ++}; ++ ++enum zxdh_context_type { ++ ZXDH_RX_READ_QPC = 1, ++ ZXDH_TX_READ_QPC, ++ ZXDH_READ_CQC, ++ ZXDH_READ_CEQC, ++ ZXDH_READ_AEQC, ++ ZXDH_RX_READ_SRQC, ++ ZXDH_READ_MRTE, ++}; ++ ++struct zxdh_context_req { ++ enum zxdh_context_type type; ++ __u32 resource_id; ++}; ++ ++#define MAX_CONTEXT_SIZE 22 ++struct zxdh_context_resp { ++ __u64 context_info[MAX_CONTEXT_SIZE]; ++ __u8 context_size; ++}; ++ ++enum zxdh_data_type { ++ ZXDH_PATH_SELECT_TYPE_CACHE = 1, ++ ZXDH_PATH_SELECT_TYPE_INDICATE, ++}; ++ ++enum zxdh_context_type_ex { ++ DATA_TYPE_PBLE_MR = 0, ++ DATA_TYPE_PBLE_SQ_RQ_SRQP_SRQ_CQ_CEQ_AEQ = 1, ++ DATA_TYPE_AH = 3, ++ DATA_TYPE_IRD = 4, ++ DATA_TYPE_TX_WINDOW = 5, ++ DATA_TYPE_SQ = 11, ++ DATA_TYPE_RQ = 13, ++ DATA_TYPE_RQ_DOORBELL_SHADOW = 14, ++ DATA_TYPE_SRQP = 15, ++ DATA_TYPE_SRQ = 16, ++ DATA_TYPE_SRQ_DOORBELL_SHADOW = 17, ++ DATA_TYPE_CQ = 18, ++ DATA_TYPE_CQ_DOORBELL_SHADOW = 19, ++ DATA_TYPE_CEQ = 20, ++ DATA_TYPE_AEQ = 21, ++ //USER DEFINED ++ DATA_TYPE_SQC = 200, ++}; ++ ++enum zxdh_error_code_const { ++ ZXDH_NOT_SUPPORT_OBJECT_ID = 100, ++ ZXDH_DMA_MEMORY_OVER_2M = 101, ++ ZXDH_DMA_READ_NOT_32_ALIGN = 102, ++ ZXDH_CACHE_ID_CHECK_ERROR = 103, ++ ZXDH_ENTRY_IDX_ERROR = 104, ++ ZXDH_PBLE_ADDRESSING_ONLY_SUPPORTS_OBJECT_NUMBER_1 = 105, ++ ZXDH_NOT_SUPPORT_TWO_LEVEL_PBLE_CODE = 106, ++ ZXDH_NOT_SUPPORT_VIRTUAL_ADDRESS = 107, ++ ZXDH_DATA_ENTRY_IDX_OVER_LIMIT = 108, ++ ZXDH_QUEUE_ID_ERROR = 109, ++ ZXDH_NOT_SUPPORT_CURRENT_DEVICE = 110, ++ /* Must be last entry*/ ++ ZXDH_CUSTOM_ERROR_CODE, ++}; ++ ++struct zxdh_get_object_data_req { ++ __u32 queue_id; ++ __u8 object_id; ++ __u32 entry_idx; ++ __u8 object_num; ++}; ++ ++struct zxdh_get_object_data_resp { ++ __u64 object_mmap_offset; ++ __u32 length; ++ __u32 object_size; ++ __u64 srqp_aligned_offset; ++ __u16 vhca_id; ++ __u8 route_id; ++ __u8 reserved; ++ __u32 queue_id; ++}; ++ ++enum zxdh_object_qp_type_const { ++ ZXDH_QP_TYPE_QP = 1, ++ ZXDH_QP_TYPE_SRQ, ++ ZXDH_QP_TYPE_CQ, ++ ZXDH_QP_TYPE_CEQ, ++ ZXDH_QP_TYPE_AEQ, ++}; ++ ++struct zxdh_object_data_print { ++ int object_id; ++ const char *object_type; ++ int src_path_select_type; ++ __u32 length; ++ __u32 queue_id; ++ int qp_type; ++}; ++ ++#define ZXDH_L2D_MPCAP_BUFF_SIZE 0x14000u ++#define ZXDH_HOST_DATA_CAP_MEM_SIZE (1024 * 1024 * 1024) ++#define ZXDH_HOST_DATA_CAP_TXRX_MEM_SIZE (2 * 1024 * 1024) ++ ++struct zxdh_cc_basic_info { ++ __u32 active_gqp_cnt; ++ __u16 active_vhca_sq_cnt; ++ __u16 active_vhca_read_cnt; ++ __u16 active_vhca_ack_cnt; ++ __u16 active_qp_sq_cur_cnt; ++ __u16 active_qp_rq_cur_cnt; ++ __u16 task_prefetch_recv_com_cnt; ++ __u64 tx_pkt_cnt; ++ __u64 rx_pkt_cnt; ++ __u16 flight_pkt_cnt; ++ __u16 retry_timeout_cnt; ++ __u16 retry_read_cnt; ++ __u16 retry_rnr_cnt; ++ __u16 retry_nak_cnt; ++ __u16 drop_read_msg_cnt; ++ __u32 tx_pkt_cnp_cnt; ++ __u32 rx_pkt_cnp_cnt; ++ __u32 tx_pkt_rtt_t1_cnt; ++ __u32 rx_pkt_rtt_t2_cnt; ++ __u32 tx_pkt_rtt_t4_cnt; ++ __u32 rx_pkt_rtt_t5_cnt; ++ __u16 limit_tx_sq_cnt; ++ __u16 limit_tx_read_ack_cnt; ++ __u32 backpres_tx_pfc_flg_pyh0_3; ++ __u32 backpres_tx_pfc_flg_pyh4_7; ++ __u16 backpres_tx_pfc_cnt; ++ __u16 rx_pkt_ecn_cnt; ++ __u8 backpres_rx_pfc_cnt; ++ __u8 backpres_rx; ++}; ++ ++/* 端口带宽统计结构 */ ++struct zxdh_get_tx_rx_bw_pps { ++ __u32 port_id; ++ __u64 tx_bytes; ++ __u64 rx_bytes; ++ __u64 tx_packets; ++ __u64 rx_packets; ++ __u64 tx_ack_cnt; ++ __u64 rx_ack_cnt; ++ __u64 tx_cnp_cnt; ++ __u64 rx_cnp_cnt; ++ __u64 tx_rtt_t1_cnt; /* 发送RTT请求报文计数 0x62065f8494[31:0] */ ++ __u64 rx_rtt_t2_cnt; /* 接收RTT请求报文计数 0x62054004f0[31:0] */ ++ __u64 tx_rtt_t4_cnt; /* 发送RTT响应报文计数 0x6205478800[25:10] */ ++ __u64 rx_rtt_t5_cnt; /* 接收RTT响应报文计数 0x62054004f4[31:0] */ ++ __u64 timestamp; ++}; ++ ++enum zxdh_health_check_reg_type { ++ ZXDH_NORMAL_REG, ++ ZXDH_WRITE_FIRST_REG, ++ ZXDH_SMMU_REG, ++}; ++ ++struct zxdh_health_check_req { ++ __u64 reg_va; ++ __u64 value_va; ++ __u64 reg_value_va_ex; ++ __u16 count; ++ __u8 reg_type : 2; ++}; ++ ++struct zxdh_health_check_resp { ++ __u16 count; ++ __u16 count_ex; ++}; ++ ++struct zxdh_reg_value { ++ __u64 reg_addr; ++ __u32 value; ++}; ++ ++enum zxdh_cfg_dev_parameter_type { ++ TX_STOP_ON_AEQ = 1, ++ RX_STOP_ON_AEQ, ++ TXRX_STOP_IOVA_CAP, ++ CLEAR_ALL_CC_BASIC_CNT, ++ CLEAR_ALL_GQPS_MP_CAP, ++}; ++ ++struct zxdh_cfg_dev_parameter_req { ++ __u8 type; ++ __u8 reserved1; ++ __u16 reserved2; ++}; ++ ++struct zxdh_db_show_res_map_req { ++ __u8 type; ++ __u32 qp_id; ++ __u64 reg_va; ++ __u64 value_va; ++ __u64 idx_va; ++ __u32 count; ++}; ++ ++struct zxdh_db_show_res_map_resp { ++ __u32 count; ++ __u64 qp_8k_index; ++}; ++ ++enum zxdh_show_res_map_type { ++ ZXDH_SHOW_RES_MAP_PF_TO_QPN, ++ ZXDH_SHOW_RES_MAP_PF_TO_VHCA, ++ ZXDH_SHOW_RES_MAP_VHCA_TO_PF, ++ ZXDH_SHOW_RES_MAP_8K_TO_GQP, ++ ZXDH_SHOW_RES_MAP_GQP_TO_VHCA_CREATED, ++ ZXDH_SHOW_RES_MAP_GQP_TO_VHCA_ACTIVE, ++ ZXDH_SHOW_RES_MAP_QP_TO_8K, ++ ZXDH_SHOW_RES_MAP_UNKNOWN, ++}; ++ ++enum hw_module { ++ HW_MODULE_TX, ++ HW_MODULE_RX, ++ HW_MODULE_CQP, ++}; ++ ++enum zxdh_ram_read_error_code_const { ++ ZXDH_CUSTOM_READ_RAM_ERROR_BASE = 100, ++ ZXDH_COPY_USER_PARAM_ERROR, ++ ZXDH_READ_RAM_ERROR, ++ ZXDH_COPY_DATA_TO_USER_ERROR, ++ /* Must be last entry*/ ++ ZXDH_CUSTOM_ERROR_UNKOWN, ++}; ++ ++struct zxdh_read_ram_req { ++ __u32 ram_id; ++ __u32 ram_addr; ++ __u32 ram_width; ++ __u32 read_count; ++ __u64 value_va; ++ __u16 hw_module; ++ __u16 reserved1; ++ __u32 reserved2; ++}; ++ ++struct zxdh_devx_qp_init_attr { ++ enum ibv_qp_type qp_type; ++ struct ibv_qp_cap cap; ++ __u32 flags; ++ __u32 reservd[2]; ++}; ++ ++struct zxdh_devx_init_qp_resp { ++ __u32 total_size; ++ __u32 rq_offset; ++ __u32 shadow_offset; ++}; ++ ++struct zxdh_devx_qp_buffers { ++ void *qp_base; ++ __u32 total_size; ++}; ++ ++struct zxdh_verbs_qp { ++#if ZXDH_HAVE_QP_EX ++ union { ++ struct ibv_qp qp; ++ struct ibv_qp_ex qp_ex; ++ }; ++#else ++ struct ibv_qp qp; ++#endif ++ uint32_t comp_mask; ++}; ++ ++struct zxdh_devx_qp { ++ struct zxdh_verbs_qp verbs_qp; ++ __u64 shadow_init_value; ++ __u32 qpn; ++ __u32 *wqe_alloc_db; ++ __u32 sqdepth; ++ __u32 rqdepth; ++ __u32 sqshift; ++ __u32 rqshift; ++ __u32 max_sq_frag_cnt; ++ __u32 max_rq_frag_cnt; ++ __u32 max_inline_data; ++ __u16 rq_signature; ++ __u16 sq_sig_all; ++ __u16 shadow_size; ++ __u8 rq_wqe_size_multiplier; ++ __u8 sq_polarity; ++ __u8 rq_polarity; ++ __u8 reserved; ++}; ++ ++struct zxdh_devx_cq_init_attr { ++ uint32_t cqe_num; ++ uint32_t flags; ++}; ++ ++struct zxdh_devx_init_cq_resp { ++ uint32_t cqe_num; ++ uint32_t total_size; ++ uint32_t shadow_offset; ++ uint32_t reservd[2]; ++}; ++ ++struct zxdh_devx_cq_buffers { ++ void *cq_base; ++ uint32_t total_size; ++ uint32_t reservd[2]; ++}; ++ ++struct zxdh_verbs_cq { ++ union { ++ struct ibv_cq cq; ++ struct ibv_cq_ex cq_ex; ++ }; ++}; ++ ++struct zxdh_devx_cq { ++ struct zxdh_verbs_cq verbs_cq; ++ uint32_t cqn; ++ uint32_t cqe_num; ++ uint8_t polarity; ++ uint32_t shadow_offset; ++ uint32_t shadow_size; ++}; ++ ++enum switch_name_e { ++ WQE_RATE_LIMIT, ++ ORD_MAX_SIZE, ++ SWITCH_NAME_MAX ++}; ++ ++enum switch_type_e { ++ CONFIG_SWITCH, ++ QUERY_SWITCH, ++ SWITCH_TYPE_MAX ++}; ++ ++struct rdma_switch_param { ++ char *ib_devname; ++ enum switch_type_e type; ++ enum switch_name_e switch_name; ++ uint32_t value; ++}; ++ ++struct config_rdma_switch_req { ++ enum switch_name_e type; ++ uint32_t value; ++}; ++ ++struct query_rdma_switch_req { ++ enum switch_name_e type; ++}; ++ ++struct query_rdma_switch_resp { ++ uint32_t value; ++ uint32_t next_reboot_value; ++}; ++ ++struct zxdh_ext_buf { ++ void *host_addr; ++ uint64_t dev_addr; ++ size_t total_size; ++}; ++ + int zxdh_get_log_trace_switch(struct ibv_context *context, + enum switch_status *status); + int zxdh_set_log_trace_switch(struct ibv_context *context, + enum switch_status status); + int zxdh_modify_qp_udp_sport(struct ibv_context *context, uint16_t udp_sport, + uint32_t qpn); ++int zxdh_cap_start(struct ibv_context *context, struct zxdh_cap_cfg *cap_cfg, ++ struct zxdh_rdma_cap_pa *cap_pa); ++int zxdh_cap_stop(struct ibv_context *context, uint8_t param); ++int zxdh_cap_free(struct ibv_context *context, uint8_t param); ++int zxdh_mp_capture(struct ibv_context *context, ++ struct zxdh_mp_cap_cfg *cap_mp_cfg, ++ struct zxdh_mp_cap_gqp *cap_mp_gqp); ++int zxdh_mp_capture_gqp(struct ibv_context *context, ++ struct zxdh_mp_cap_gqp_cfg *cfg, ++ struct zxdh_mp_cap_gqp *cap_mp_gqp); ++int zxdh_mp_get_data(struct ibv_context *context, uint8_t param); ++int zxdh_mp_capture_clear(struct ibv_context *context, ++ struct zxdh_cap_gqp *cap_gqp); ++int zxdh_get_active_vhca_gqps(struct ibv_context *context, ++ struct zxdh_active_vhca_gqps *resp); ++int zxdh_get_cc_basic_info(struct ibv_context *context, ++ struct zxdh_cc_basic_info *resp); + int zxdh_query_qpc(struct ibv_qp *qp, struct zxdh_rdma_qpc *qpc); + int zxdh_modify_qpc(struct ibv_qp *qp, struct zxdh_rdma_qpc *qpc, + uint64_t qpc_mask); + int zxdh_reset_qp(struct ibv_qp *qp, uint64_t opcode); ++int zxdh_rdma_hmc_query(struct ibv_context *context, ++ struct zxdh_context_req *req, ++ struct zxdh_context_resp *resp); ++int zxdh_rdma_switch_query(struct ibv_context *context, ++ struct query_rdma_switch_req *req, ++ struct query_rdma_switch_resp *resp); ++int zxdh_rdma_switch_config(struct ibv_context *context, ++ struct config_rdma_switch_req *req); ++int zxdh_get_object_data(struct ibv_context *context, ++ struct zxdh_get_object_data_req *req, ++ struct zxdh_get_object_data_resp *resp); ++int zxdh_rdma_health_check(struct ibv_context *context, ++ struct zxdh_health_check_req *req, ++ struct zxdh_health_check_resp *resp); ++int zxdh_cfg_dev_parameter(struct ibv_context *context, ++ struct zxdh_cfg_dev_parameter_req *req); ++int zxdh_show_res_map(struct ibv_context *context, ++ struct zxdh_db_show_res_map_req *req, ++ struct zxdh_db_show_res_map_resp *resp); ++int zxdh_read_ram(struct ibv_context *context, struct zxdh_read_ram_req *req); + ++int zxdh_devx_get_qp_init_size(struct ibv_context *ibv_ctx, ++ struct zxdh_devx_qp_init_attr *attr, ++ struct zxdh_devx_init_qp_resp *resp); ++struct zxdh_devx_qp *zxdh_devx_create_qp(struct ibv_pd *pd, ++ struct ibv_qp_init_attr *attr, ++ struct zxdh_devx_qp_buffers *buffer); ++int zxdh_devx_destroy_qp(struct zxdh_devx_qp *devx_qp); ++int zxdh_devx_modify_qp(struct zxdh_devx_qp *devx_qp, struct ibv_qp_attr *attr, ++ int attr_mask); ++int zxdh_set_qp_credit_flag(struct ibv_context *context, ++ struct ibv_qp *qp, uint64_t credit_flag); ++int zxdh_devx_get_cq_init_size(struct ibv_context *context, ++ struct zxdh_devx_cq_init_attr *attr, ++ struct zxdh_devx_init_cq_resp *resp); ++struct zxdh_devx_cq *zxdh_devx_create_cq(struct ibv_context *context, ++ int cqe_num, ++ struct ibv_comp_channel *channel, ++ int comp_vector, ++ struct zxdh_devx_cq_buffers *buffer); ++int zxdh_devx_destroy_cq(struct zxdh_devx_cq *cq); ++int zxdh_devx_modify_cq(struct ibv_cq *cq, uint32_t overflow_check_en); ++struct ibv_mr *zxdh_devx_reg_mr(struct ibv_pd *pd, void *host_addr, size_t length, uint64_t dev_addr, unsigned int access); ++int zxdh_devx_dereg_mr(struct ibv_mr *mr); ++struct ibv_cq *zxdh_devx_create_cq_with_ext_buf(struct ibv_context *context, ++ int cqe_num, ++ struct ibv_comp_channel *channel, ++ int comp_vector, ++ struct zxdh_ext_buf *buf); ++struct ibv_qp *zxdh_devx_create_qp_with_ext_buf(struct ibv_pd *pd, ++ struct ibv_qp_init_attr *attr, ++ struct zxdh_ext_buf *buf); ++int zxdh_devx_get_peer_dev_sbdf(struct ibv_context *ibv_ctx, uint64_t *peer_pci_bdf); + #ifdef __cplusplus + } + #endif +diff -ruN baseline/providers/zrdma/zxdh_hw.c b/providers/zrdma/zxdh_hw.c +--- a/providers/zrdma/zxdh_hw.c 2026-08-06 15:37:55.353862831 +0800 ++++ b/providers/zrdma/zxdh_hw.c 2026-08-06 15:29:37.681705521 +0800 +@@ -3,37 +3,152 @@ + #include "zxdh_status.h" + #include "zxdh_defs.h" + #include "zxdh_verbs.h" +-#include "main.h" ++#include "zxdh_zrdma.h" + #include + #include + #include "private_verbs_cmd.h" + #include + #include + #include +-#include + #include +-#include +-#include +-#include +-#include +-#include +-#include + #define ERROR_CODE_VALUE 65 + +-static void qp_tx_psn_add(__u32 *x, __u32 y, __u16 mtu) ++#if defined(__x86_64__) || defined(_M_X64) ++ #include // SSE2 for _mm_stream_si128 ++ #include // AVX/AVX2 for _mm_stream_si256 ++ #include // for _mm_stream_si512 ++ #define USE_X86_AVX 1 ++#endif ++ ++#if defined(USE_X86_AVX) // x86平台实现 ++int stream_memcpy(void *dest, const void *src, size_t n); ++ ++int stream_memcpy(void *dest, const void *src, size_t n) ++{ ++ size_t i = 0; ++ int ret_code = 0; ++ char *d = NULL; ++ const char *s = NULL; ++ if (n < 32 || dest == NULL || src == NULL || n % 32 != 0 || ++ (unsigned long)dest % 32 != 0 || (unsigned long)src % 32 != 0) { ++ ret_code = -EINVAL; ++ return ret_code; ++ } ++ d = (char *)dest; ++ s = (const char *)src; ++ ++ for (i = n - 32; i > 0; i -= 32) { ++ __m256i data = _mm256_load_si256((const __m256i *)&s[i]); ++ _mm256_stream_si256((__m256i *)&d[i], data); ++ } ++ if (n != 32) ++ udma_to_device_barrier(); ++ __m256i data = _mm256_load_si256((const __m256i *)&s[0]); ++ _mm256_stream_si256((__m256i *)&d[0], data); ++ _mm_sfence(); ++ return ret_code; ++} ++#endif ++ ++static enum zxdh_status_code zxdh_copy_wqe_to_l2d_if_ext_qp(struct zxdh_qp *qp, ++ __u32 wqe_idx, __le64 *wqe, ++ __u64 len) ++{ ++ if (!qp || !wqe || len == 0) ++ return ZXDH_ERR_MEMCPY_FAILED; ++ if (qp->is_ext_qp) { ++#if defined(USE_X86_AVX) ++ if (stream_memcpy(qp->ext_sq_base_va[wqe_idx].elem, wqe, len)) ++ return ZXDH_ERR_MEMCPY_FAILED; ++#else ++ zxdh_status_to_errno_log(ZXDH_NOT_SUPPORTED, ++ "blue_flames only support x86 arch\n"); ++ return ZXDH_NOT_SUPPORTED; ++#endif ++ } ++ return ZXDH_SUCCESS; ++} ++ ++static enum zxdh_status_code zxdh_copy_rq_wqe_to_l2d_if_ext_qp(struct zxdh_qp *qp, ++ __u32 wqe_idx, __le64 *wqe, ++ __u64 len) ++{ ++ if (qp == NULL || wqe == NULL || len == 0) { ++ return ZXDH_ERR_MEMCPY_FAILED; ++ } ++ if (qp->is_ext_qp) { ++#if defined(USE_X86_AVX) ++ if (stream_memcpy(qp->ext_rq_base_va[wqe_idx * qp->rq_wqe_size_multiplier].elem, wqe, len)) ++ return ZXDH_ERR_MEMCPY_FAILED; ++#else ++ zxdh_status_to_errno_log(ZXDH_NOT_SUPPORTED, ++ "blue_flames only support x86 arch\n"); ++ return ZXDH_NOT_SUPPORTED; ++#endif ++ } ++ return ZXDH_SUCCESS; ++} ++ ++static inline void qp_tx_psn_add(__u32 *x, __u32 y, __u16 mtu) + { + if (y == 0) { + *x = (*x + 1) & 0xffffff; + return; + } +- *x = (*x + ((y % mtu) ? (y / mtu + 1) : y / mtu)) & 0xffffff; ++ __u32 chunks = (y + mtu - 1) / mtu; ++ *x = (*x + chunks) & 0xffffff; + } + +-int zxdh_get_write_imm_split_switch(void) ++/** ++ * zxdh_move_ring_tail_quanta - move ring tail by calculated quanta count ++ * @qp: hw qp ptr ++ * @wqe_idx: current wqe index ++ * @quanta: quanta count to move (for SQ, derived from wrtrk) ++ * ++ * This function calculates the number of quanta to move considering ring ++ * wrap-around scenarios, then moves the ring tail with lock protection. ++ */ ++static enum zxdh_status_code zxdh_move_ring_tail_quanta(struct zxdh_qp *qp, ++ __u32 wqe_idx, ++ __u32 quanta) ++{ ++ __u32 count; ++ ++ /* 计算需要移动的quanta数（考虑卷绕） */ ++ if (wqe_idx >= qp->last_sqe_idx) { ++ count = wqe_idx - qp->last_sqe_idx + quanta; ++ } else { ++ /* 卷绕场景 */ ++ count = (qp->sq_ring.size - qp->last_sqe_idx) + wqe_idx + quanta; ++ } ++ ++ if (unlikely(pthread_spin_lock(qp->quanta_lock))) ++ return ZXDH_ERR_REG_QSET; ++ ++ ZXDH_RING_MOVE_TAIL_BY_COUNT(qp->sq_ring, count); ++ qp->last_sqe_idx = (wqe_idx + quanta) % qp->sq_ring.size; ++ if (unlikely(pthread_spin_unlock(qp->quanta_lock))) ++ return ZXDH_ERR_REG_QSET; ++ ++ zxdh_quanta_dbg("[MOVE_TAIL]: wqe_idx=%u qp_num=%u sq_ring.tail=%u last_sqe_idx=%u sq_ring.size=%u count=%u used_quanta_cnt=%u\n", ++ wqe_idx, qp->qp_id, qp->sq_ring.tail, qp->last_sqe_idx, qp->sq_ring.size, count, qp->sq_ring.used_quanta_cnt); ++ return ZXDH_SUCCESS; ++} ++ ++ ++ ++/** ++ * zxdh_fragcnt_to_quanta_sq - calculate quanta based on fragment count for SQ ++ * @frag_cnt: number of fragments ++ * @quanta: quanta for frag_cnt ++ */ ++static inline enum zxdh_status_code zxdh_fragcnt_to_quanta_sq(__u32 frag_cnt, ++ __u16 *quanta) + { +- char *env; +- env = getenv("ZXDH_WRITE_IMM_SPILT_ENABLE"); +- return (env != NULL) ? atoi(env) : 0; ++ if (unlikely(frag_cnt > ZXDH_MAX_SQ_FRAG)) ++ return ZXDH_ERR_INVALID_FRAG_COUNT; ++ *quanta = (frag_cnt >> 1) + 1; ++ return 0; + } + + /** +@@ -92,27 +207,17 @@ + + set_64bit_val(wqe, 0, hdr); + +- return 0; ++ return zxdh_copy_wqe_to_l2d_if_ext_qp(qp, wqe_idx, wqe, ZXDH_QP_SQE_MIN_SIZE); + } + +-/** +- * zxdh_clr_wqes - clear next 128 sq entries +- * @qp: hw qp ptr +- * @qp_wqe_idx: wqe_idx +- */ +-void zxdh_clr_wqes(struct zxdh_qp *qp, __u32 qp_wqe_idx) ++void read_wqe_ctrl_hdr_coherent_if_need(struct zxdh_qp *qp, __le64 *wqe) + { +- __le64 *wqe; +- __u32 wqe_idx; ++ volatile __u64 tmp; + +- if (!(qp_wqe_idx & 0x7F)) { +- wqe_idx = (qp_wqe_idx + 128) % qp->sq_ring.size; +- wqe = qp->sq_base[wqe_idx].elem; +- if (wqe_idx) +- memset(wqe, qp->swqe_polarity ? 0 : 0xFF, 0x1000); +- else +- memset(wqe, qp->swqe_polarity ? 0xFF : 0, 0x1000); +- } ++ if (qp == NULL || wqe == NULL || !qp->ext_buf_mode) ++ return; ++ ++ get_64bit_volatile_val(wqe, 0, &tmp); + } + + /** +@@ -123,7 +228,6 @@ + { + /* valid bit is written before ringing doorbell */ + udma_to_device_barrier(); +- + db_wr32(qp->qp_id, qp->wqe_alloc_db); + qp->initial_ring.head = qp->sq_ring.head; + } +@@ -135,7 +239,17 @@ + void zxdh_qp_set_shadow_area(struct zxdh_qp *qp) + { + __u8 polarity = 0; ++#if 0 ++ struct ibv_qp *ib_qp; ++ struct zxdh_uqp *iwuqp; ++ struct zxdh_uvcontext *iwvctx; + ++ iwuqp = container_of(qp, struct zxdh_uqp, qp); ++ iwvctx = container_of(iwuqp->vqp.qp.context, struct zxdh_uvcontext, ++ ibv_ctx.context); ++ ++ ib_qp = &iwuqp->vqp.qp; ++#endif + polarity = ((ZXDH_RING_CURRENT_HEAD(qp->rq_ring) == 0) ? + !qp->rwqe_polarity : + qp->rwqe_polarity); +@@ -143,6 +257,13 @@ + FIELD_PREP(ZXDHQPDBSA_RQ_POLARITY, polarity) | + FIELD_PREP(ZXDHQPDBSA_RQ_SW_HEAD, + ZXDH_RING_CURRENT_HEAD(qp->rq_ring))); ++#if 0 ++ if (iwvctx->dev_attrs.chip_rev < 2 && ++ !qp->credit_set && !zxdh_set_qp_credit_flag(&iwvctx->ibv_ctx.context, ib_qp, 1)) ++ { ++ qp->credit_set = true; ++ } ++#endif + } + + /** +@@ -168,6 +289,7 @@ + if (ZXDH_RING_CURRENT_HEAD(qp->initial_ring) != + ZXDH_RING_CURRENT_TAIL(qp->sq_ring) && + !qp->push_mode) { ++ read_wqe_ctrl_hdr_coherent_if_need(qp, wqe); + if (post_sq) + zxdh_qp_post_wr(qp); + } else { +@@ -197,19 +319,22 @@ + avail_quanta = ZXDH_MAX_SQ_WQES_PER_PAGE - + (ZXDH_RING_CURRENT_HEAD(qp->sq_ring) % + ZXDH_MAX_SQ_WQES_PER_PAGE); +- if (quanta <= avail_quanta) { ++ if (likely(quanta <= avail_quanta)) { + /* WR fits in current chunk */ +- if (quanta > ZXDH_SQ_RING_FREE_QUANTA(qp->sq_ring)) ++ if (unlikely(quanta > ZXDH_RING_FREE_QUANTA(qp->sq_ring))) + return NULL; + } else { + /* Need to pad with NOP */ +- if (quanta + avail_quanta > +- ZXDH_SQ_RING_FREE_QUANTA(qp->sq_ring)) ++ if (quanta + avail_quanta > ZXDH_RING_FREE_QUANTA(qp->sq_ring)) + return NULL; + + for (i = 0; i < avail_quanta; i++) { + zxdh_nop_1(qp); ++ if (unlikely(pthread_spin_lock(qp->quanta_lock))) ++ return NULL; + ZXDH_RING_MOVE_HEAD_NOCHECK(qp->sq_ring); ++ if (unlikely(pthread_spin_unlock(qp->quanta_lock))) ++ return NULL; + } + } + +@@ -217,7 +342,13 @@ + if (!*wqe_idx) + qp->swqe_polarity = !qp->swqe_polarity; + ++ if (unlikely(pthread_spin_lock(qp->quanta_lock))) ++ return NULL; + ZXDH_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, quanta); ++ if (unlikely(pthread_spin_unlock(qp->quanta_lock))) ++ return NULL; ++ zxdh_quanta_dbg("[MOVE_HEAD]: qp_num=%u sq_ring.head=%u last_sqe_idx=%u quanta=%u\n", ++ qp->qp_id, qp->sq_ring.head, qp->last_sqe_idx, quanta); + + wqe = qp->sq_base[*wqe_idx].elem; + qp->sq_wrtrk_array[*wqe_idx].wrid = info->wr_id; +@@ -240,9 +371,17 @@ + if (ZXDH_RING_FULL_ERR(qp->rq_ring)) + return NULL; + ++ if (unlikely(pthread_spin_lock(qp->quanta_lock))) ++ return NULL; + ZXDH_ATOMIC_RING_MOVE_HEAD(qp->rq_ring, *wqe_idx, ret_code); +- if (ret_code) ++ if (ret_code) { ++ pthread_spin_unlock(qp->quanta_lock); ++ return NULL; ++ } ++ if (unlikely(pthread_spin_unlock(qp->quanta_lock))) + return NULL; ++ zxdh_quanta_dbg("[MOVE_HEAD]: qp_num=%u rq_ring.head=%u last_rqe_idx=%u\n", ++ qp->qp_id, qp->rq_ring.head, qp->last_rqe_idx); + + if (!*wqe_idx) + qp->rwqe_polarity = !qp->rwqe_polarity; +@@ -287,8 +426,6 @@ + if (!wqe) + return ZXDH_ERR_QP_TOOMANY_WRS_POSTED; + +- zxdh_clr_wqes(qp, wqe_idx); +- + if (op_info->num_lo_sges) { + set_64bit_val( + wqe, 16, +@@ -394,6 +531,10 @@ + udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */ + + set_64bit_val(wqe, 0, hdr); ++ read_wqe_ctrl_hdr_coherent_if_need(qp, wqe); ++ ret_code = zxdh_copy_wqe_to_l2d_if_ext_qp(qp, wqe_idx, wqe, ZXDH_QP_SQE_MIN_SIZE * quanta); ++ if (ret_code) ++ return ret_code; + if (post_sq) + zxdh_qp_post_wr(qp); + qp_tx_psn_add(&qp->next_psn, total_size, qp->mtu); +@@ -635,8 +776,6 @@ + if (!wqe) + return ZXDH_ERR_QP_TOOMANY_WRS_POSTED; + +- zxdh_clr_wqes(qp, wqe_idx); +- + addl_frag_cnt = + op_info->num_lo_sges > 1 ? (op_info->num_lo_sges - 1) : 0; + local_fence |= info->local_fence; +@@ -707,6 +846,12 @@ + udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */ + + set_64bit_val(wqe, 0, hdr); ++ read_wqe_ctrl_hdr_coherent_if_need(qp, wqe); ++ ++ ret_code = zxdh_copy_wqe_to_l2d_if_ext_qp(qp, wqe_idx, wqe, ZXDH_QP_SQE_MIN_SIZE * quanta); ++ if (ret_code) ++ return ret_code; ++ + if (post_sq) + zxdh_qp_post_wr(qp); + return 0; +@@ -727,8 +872,12 @@ + struct zxdh_post_sq_info split_part2_info = { 0 }; + struct zxdh_rdma_read *op_info; + enum zxdh_status_code ret_code; ++ struct zxdh_uqp *iwuqp; ++ struct zxdh_uvcontext *iwvctx; + __u32 i, total_size = 0, pre_cal_psn = 0; +- ++ iwuqp = container_of(qp, struct zxdh_uqp, qp); ++ iwvctx = container_of(iwuqp->vqp.qp.context, struct zxdh_uvcontext, ++ ibv_ctx.context); + op_info = &info->op.rdma_read; + if (qp->max_sq_frag_cnt < op_info->num_lo_sges) + return ZXDH_ERR_INVALID_FRAG_COUNT; +@@ -744,7 +893,8 @@ + op_info->rem_addr.len = total_size; + pre_cal_psn = qp->next_psn; + qp_tx_psn_add(&pre_cal_psn, total_size, qp->mtu); +- if (read_wqe_need_split(pre_cal_psn, qp->next_psn)) { ++ if (read_wqe_need_split(pre_cal_psn, qp->next_psn, ++ iwvctx->dev_attrs.chip_rev)) { + split_two_part_info(qp, info, qp->next_psn, pre_cal_psn, + &split_part1_info, &split_part2_info); + ret_code = zxdh_post_rdma_read(qp, &split_part1_info, post_sq, +@@ -817,8 +967,6 @@ + if (!wqe) + return ZXDH_ERR_QP_TOOMANY_WRS_POSTED; + +- zxdh_clr_wqes(qp, wqe_idx); +- + read_fence |= info->read_fence; + addl_frag_cnt = op_info->num_sges > 1 ? (op_info->num_sges - 1) : 0; + if (op_info->num_sges) { +@@ -927,6 +1075,10 @@ + udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */ + + set_64bit_val(wqe, 0, hdr); ++ read_wqe_ctrl_hdr_coherent_if_need(qp, wqe); ++ ret_code = zxdh_copy_wqe_to_l2d_if_ext_qp(qp, wqe_idx, wqe, ZXDH_QP_SQE_MIN_SIZE * quanta); ++ if (ret_code) ++ return ret_code; + if (post_sq) + zxdh_qp_post_wr(qp); + qp_tx_psn_add(&qp->next_psn, total_size, qp->mtu); +@@ -975,22 +1127,26 @@ + if (ret_code) + return ret_code; + +- if (quanta > ZXDH_SQ_RING_FREE_QUANTA(qp->sq_ring)) ++ if (quanta > ZXDH_RING_FREE_QUANTA(qp->sq_ring)) + return ZXDH_ERR_QP_TOOMANY_WRS_POSTED; + + wqe_idx = ZXDH_RING_CURRENT_HEAD(qp->sq_ring); + if (!wqe_idx) + qp->swqe_polarity = !qp->swqe_polarity; + ++ if (unlikely(pthread_spin_lock(qp->quanta_lock))) ++ return ZXDH_ERR_REG_QSET; + ZXDH_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, quanta); ++ if (unlikely(pthread_spin_unlock(qp->quanta_lock))) ++ return ZXDH_ERR_REG_QSET; ++ zxdh_quanta_dbg("[MOVE_HEAD]: qp_num=%u sq_ring.head=%u last_sqe_idx=%u quanta=%u\n", ++ qp->qp_id, qp->sq_ring.head, qp->last_sqe_idx, quanta); + + wqe_base = qp->sq_base[wqe_idx].elem; + qp->sq_wrtrk_array[wqe_idx].wrid = info->wr_id; + qp->sq_wrtrk_array[wqe_idx].wr_len = total_size; + qp->sq_wrtrk_array[wqe_idx].quanta = quanta; + +- zxdh_clr_wqes(qp, wqe_idx); +- + read_fence |= info->read_fence; + addl_frag_cnt = op_info->num_sges > 1 ? (op_info->num_sges - 1) : 0; + hdr = FIELD_PREP(ZXDHQPSQ_VALID, qp->swqe_polarity) | +@@ -1106,6 +1262,10 @@ + udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */ + + set_64bit_val(wqe_base, 0, hdr); ++ read_wqe_ctrl_hdr_coherent_if_need(qp, wqe_base); ++ ret_code = zxdh_copy_wqe_to_l2d_if_ext_qp(qp, wqe_idx, wqe_base, ZXDH_QP_SQE_MIN_SIZE * quanta); ++ if (ret_code) ++ return ret_code; + if (post_sq) + zxdh_qp_post_wr(qp); + +@@ -1265,6 +1425,7 @@ + __u32 wqe_idx; + bool read_fence = false; + __u16 quanta; ++ enum zxdh_status_code ret_code; + bool imm_data_flag = info->imm_data_valid ? 1 : 0; + + op_info = &info->op.inline_rdma_write; +@@ -1281,8 +1442,6 @@ + if (!wqe) + return ZXDH_ERR_QP_TOOMANY_WRS_POSTED; + +- zxdh_clr_wqes(qp, wqe_idx); +- + read_fence |= info->read_fence; + hdr = FIELD_PREP(ZXDHQPSQ_VALID, qp->swqe_polarity) | + FIELD_PREP(ZXDHQPSQ_OPCODE, info->op_type) | +@@ -1293,7 +1452,7 @@ + FIELD_PREP(ZXDHQPSQ_IMMDATAFLAG, imm_data_flag) | + FIELD_PREP(ZXDHQPSQ_WRITE_INLINEDATAFLAG, 1) | + FIELD_PREP(ZXDHQPSQ_WRITE_INLINEDATALEN, op_info->len) | +- FIELD_PREP(ZXDHQPSQ_ADDFRAGCNT, quanta - 1) | ++ FIELD_PREP(ZXDHQPSQ_ADDFRAGCNT, (__u16)(quanta - 1)) | + FIELD_PREP(ZXDHQPSQ_REMSTAG, op_info->rem_addr.stag); + set_64bit_val(wqe, 24, + FIELD_PREP(ZXDHQPSQ_FRAG_TO, op_info->rem_addr.tag_off)); +@@ -1315,7 +1474,10 @@ + udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */ + + set_64bit_val(wqe, 0, hdr); +- ++ read_wqe_ctrl_hdr_coherent_if_need(qp, wqe); ++ ret_code = zxdh_copy_wqe_to_l2d_if_ext_qp(qp, wqe_idx, wqe, ZXDH_QP_SQE_MIN_SIZE * quanta); ++ if (ret_code) ++ return ret_code; + if (post_sq) + zxdh_qp_post_wr(qp); + qp_tx_psn_add(&qp->next_psn, op_info->len, qp->mtu); +@@ -1340,6 +1502,7 @@ + bool read_fence = false; + __u16 quanta; + bool imm_data_flag = info->imm_data_valid ? 1 : 0; ++ enum zxdh_status_code ret_code; + + op_info = &info->op.inline_rdma_send; + +@@ -1355,8 +1518,6 @@ + if (!wqe) + return ZXDH_ERR_QP_TOOMANY_WRS_POSTED; + +- zxdh_clr_wqes(qp, wqe_idx); +- + read_fence |= info->read_fence; + hdr = FIELD_PREP(ZXDHQPSQ_VALID, qp->swqe_polarity) | + FIELD_PREP(ZXDHQPSQ_OPCODE, info->op_type) | +@@ -1364,7 +1525,7 @@ + FIELD_PREP(ZXDHQPSQ_LOCALFENCE, info->local_fence) | + FIELD_PREP(ZXDHQPSQ_READFENCE, read_fence) | + FIELD_PREP(ZXDHQPSQ_SOLICITED, info->solicited) | +- FIELD_PREP(ZXDHQPSQ_ADDFRAGCNT, quanta - 1) | ++ FIELD_PREP(ZXDHQPSQ_ADDFRAGCNT, (__u16)(quanta - 1)) | + FIELD_PREP(ZXDHQPSQ_IMMDATAFLAG, imm_data_flag) | + FIELD_PREP(ZXDHQPSQ_REMSTAG, info->stag_to_inv); + set_64bit_val(wqe, 24, +@@ -1388,7 +1549,10 @@ + udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */ + + set_64bit_val(wqe, 0, hdr); +- ++ read_wqe_ctrl_hdr_coherent_if_need(qp, wqe); ++ ret_code = zxdh_copy_wqe_to_l2d_if_ext_qp(qp, wqe_idx, wqe, ZXDH_QP_SQE_MIN_SIZE * quanta); ++ if (ret_code) ++ return ret_code; + if (post_sq) + zxdh_qp_post_wr(qp); + +@@ -1419,6 +1583,7 @@ + __u32 inline_len; + __u32 copy_size; + __u8 *inline_valid_addr; ++ enum zxdh_status_code ret_code; + + op_info = &info->op.inline_rdma_send; + inline_len = op_info->len; +@@ -1430,22 +1595,26 @@ + + quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(op_info->len, + imm_data_flag); +- if (quanta > ZXDH_SQ_RING_FREE_QUANTA(qp->sq_ring)) ++ if (quanta > ZXDH_RING_FREE_QUANTA(qp->sq_ring)) + return ZXDH_ERR_QP_TOOMANY_WRS_POSTED; + + wqe_idx = ZXDH_RING_CURRENT_HEAD(qp->sq_ring); + if (!wqe_idx) + qp->swqe_polarity = !qp->swqe_polarity; + ++ if (unlikely(pthread_spin_lock(qp->quanta_lock))) ++ return ZXDH_ERR_REG_QSET; + ZXDH_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, quanta); ++ if (unlikely(pthread_spin_unlock(qp->quanta_lock))) ++ return ZXDH_ERR_REG_QSET; ++ zxdh_quanta_dbg("[MOVE_HEAD]: qp_num=%u sq_ring.head=%u last_sqe_idx=%u quanta=%u\n", ++ qp->qp_id, qp->sq_ring.head, qp->last_sqe_idx, quanta); + + wqe_base = qp->sq_base[wqe_idx].elem; + qp->sq_wrtrk_array[wqe_idx].wrid = info->wr_id; + qp->sq_wrtrk_array[wqe_idx].wr_len = op_info->len; + qp->sq_wrtrk_array[wqe_idx].quanta = quanta; + +- zxdh_clr_wqes(qp, wqe_idx); +- + read_fence |= info->read_fence; + hdr = FIELD_PREP(ZXDHQPSQ_VALID, qp->swqe_polarity) | + FIELD_PREP(ZXDHQPSQ_OPCODE, info->op_type) | +@@ -1454,7 +1623,7 @@ + FIELD_PREP(ZXDHQPSQ_IMMDATAFLAG, imm_data_flag) | + FIELD_PREP(ZXDHQPSQ_UD_INLINEDATAFLAG, 1) | + FIELD_PREP(ZXDHQPSQ_UD_INLINEDATALEN, op_info->len) | +- FIELD_PREP(ZXDHQPSQ_UD_ADDFRAGCNT, quanta - 1) | ++ FIELD_PREP(ZXDHQPSQ_UD_ADDFRAGCNT, (__u16)(quanta - 1)) | + FIELD_PREP(ZXDHQPSQ_AHID, op_info->ah_id); + set_64bit_val(wqe_base, 24, + FIELD_PREP(ZXDHQPSQ_DESTQPN, op_info->dest_qp) | +@@ -1542,7 +1711,10 @@ + udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */ + + set_64bit_val(wqe_base, 0, hdr); +- ++ read_wqe_ctrl_hdr_coherent_if_need(qp, wqe_base); ++ ret_code = zxdh_copy_wqe_to_l2d_if_ext_qp(qp, wqe_idx, wqe_base, ZXDH_QP_SQE_MIN_SIZE * quanta); ++ if (ret_code) ++ return ret_code; + if (post_sq) + zxdh_qp_post_wr(qp); + +@@ -1564,6 +1736,7 @@ + __u64 hdr; + __u32 wqe_idx; + bool local_fence = true; ++ enum zxdh_status_code ret_code; + + op_info = &info->op.inv_local_stag; + +@@ -1572,8 +1745,6 @@ + if (!wqe) + return ZXDH_ERR_QP_TOOMANY_WRS_POSTED; + +- zxdh_clr_wqes(qp, wqe_idx); +- + set_64bit_val(wqe, 16, 0); + + hdr = FIELD_PREP(ZXDHQPSQ_VALID, qp->swqe_polarity) | +@@ -1586,7 +1757,11 @@ + udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */ + + set_64bit_val(wqe, 0, hdr); ++ read_wqe_ctrl_hdr_coherent_if_need(qp, wqe); + ++ ret_code = zxdh_copy_wqe_to_l2d_if_ext_qp(qp, wqe_idx, wqe, ZXDH_QP_SQE_MIN_SIZE); ++ if (ret_code) ++ return ret_code; + if (post_sq) + zxdh_qp_post_wr(qp); + +@@ -1609,6 +1784,7 @@ + bool local_fence = true; + __u8 access = 1; + __u16 value = 0; ++ enum zxdh_status_code ret_code; + + op_info = &info->op.bind_window; + local_fence |= info->local_fence; +@@ -1618,8 +1794,6 @@ + if (!wqe) + return ZXDH_ERR_QP_TOOMANY_WRS_POSTED; + +- zxdh_clr_wqes(qp, wqe_idx); +- + if (op_info->ena_writes) { + access = (op_info->ena_reads << 2) | + (op_info->ena_writes << 3) | (1 << 1) | access; +@@ -1652,10 +1826,12 @@ + udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */ + + set_64bit_val(wqe, 0, hdr); +- ++ read_wqe_ctrl_hdr_coherent_if_need(qp, wqe); ++ ret_code = zxdh_copy_wqe_to_l2d_if_ext_qp(qp, wqe_idx, wqe, ZXDH_QP_SQE_MIN_SIZE); ++ if (ret_code) ++ return ret_code; + if (post_sq) + zxdh_qp_post_wr(qp); +- + return 0; + } + +@@ -1720,8 +1896,9 @@ + zxdh_sleep_ns(1000); + + set_64bit_val(wqe, 8, FIELD_PREP(ZXDHQPRQ_VALID, qp->rwqe_polarity)); ++ ++ return zxdh_copy_rq_wqe_to_l2d_if_ext_qp(qp, wqe_idx, wqe, qp->rq_wqe_size_multiplier * ZXDH_QP_FRAG_BYTESIZE); + +- return 0; + } + + /** +@@ -1818,19 +1995,26 @@ + } + return; + } +- if (info->major_err == ZXDH_RETRY_ACK_MAJOR_ERR && +- info->minor_err == ZXDH_RETRY_ACK_MINOR_ERR) { +- info->comp_status = ZXDH_COMPL_STATUS_RETRY_ACK_ERR; ++ ++ switch (info->major_err) { ++ case ZXDH_RETRY_ACK_MAJOR_ERR: ++ if (info->minor_err == ZXDH_RETRY_ACK_MINOR_ERR) { ++ info->comp_status = ZXDH_COMPL_STATUS_RETRY_ACK_ERR; ++ return; ++ } ++ if (info->minor_err == ZXDH_TX_WINDOW_QUERY_ITEM_MINOR_ERR) { ++ info->comp_status = ++ ZXDH_COMPL_STATUS_TX_WINDOW_QUERY_ITEM_ERR; ++ return; ++ } ++ break; ++ case ZXDH_FLUSH_MAJOR_ERR: ++ info->comp_status = ZXDH_COMPL_STATUS_FLUSHED; + return; +- } +- if (info->major_err == ZXDH_RETRY_ACK_MAJOR_ERR && +- info->minor_err == ZXDH_TX_WINDOW_QUERY_ITEM_MINOR_ERR) { +- info->comp_status = ZXDH_COMPL_STATUS_TX_WINDOW_QUERY_ITEM_ERR; ++ default: ++ info->comp_status = ZXDH_COMPL_STATUS_UNKNOWN; + return; + } +- info->comp_status = (info->major_err == ZXDH_FLUSH_MAJOR_ERR) ? +- ZXDH_COMPL_STATUS_FLUSHED : +- ZXDH_COMPL_STATUS_UNKNOWN; + } + + __le64 *get_current_cqe(struct zxdh_cq *cq) +@@ -1849,7 +2033,11 @@ + info->imm_valid = (bool)FIELD_GET(ZXDH_CQ_IMMVALID, qword2); + if (info->imm_valid) { + info->imm_data = (__u32)FIELD_GET(ZXDH_CQ_IMMDATA, qword3); +- info->op_type = ZXDH_OP_TYPE_REC_IMM; ++ if (info->op_type == ZXDH_OP_TYPE_WRITE_WITH_IMM) { ++ info->op_type = ZXDH_OP_TYPE_REC_WRITE_IMM; ++ } else { ++ info->op_type = ZXDH_OP_TYPE_REC_IMM; ++ } + } else { + info->op_type = ZXDH_OP_TYPE_REC; + } +@@ -1870,16 +2058,21 @@ + } + } + +-static void update_cq_poll_info(struct zxdh_qp *qp, +- struct zxdh_cq_poll_info *info, __u32 wqe_idx, +- __u64 qword0) ++static enum zxdh_status_code update_cq_poll_info(struct zxdh_qp *qp, ++ struct zxdh_cq_poll_info *info, ++ __u32 wqe_idx, __u64 qword0) + { ++ __u32 quanta; ++ enum zxdh_status_code ret; + info->wr_id = qp->sq_wrtrk_array[wqe_idx].wrid; + if (!info->comp_status) + info->bytes_xfered = qp->sq_wrtrk_array[wqe_idx].wr_len; + info->op_type = (__u8)FIELD_GET(ZXDHCQ_OP, qword0); +- ZXDH_RING_SET_TAIL(qp->sq_ring, +- wqe_idx + qp->sq_wrtrk_array[wqe_idx].quanta); ++ quanta = qp->sq_wrtrk_array[wqe_idx].quanta; ++ ret = zxdh_move_ring_tail_quanta(qp, wqe_idx, quanta); ++ if (ret) ++ return ret; ++ return ZXDH_SUCCESS; + } + + static enum zxdh_status_code +@@ -1895,9 +2088,9 @@ + ib_qp = &iwuqp->vqp.qp; + ret = zxdh_query_qpc(ib_qp, &qpc); + if (ret) { +- verbs_err(verbs_get_ctx(ib_qp->context), +- "process tx window query item query qpc failed:%d\n", +- ret); ++ zxdh_dbg(ZXDH_DBG_QP, ++ "process tx window query item query qpc failed:%d\n", ++ ret); + return ZXDH_ERR_RETRY_ACK_ERR; + } + if (qpc.tx_last_ack_psn != qp->qp_last_ack_qsn) +@@ -1909,9 +2102,9 @@ + + ret = zxdh_reset_qp(ib_qp, ZXDH_RESET_RETRY_TX_ITEM_FLAG); + if (ret) { +- verbs_err(verbs_get_ctx(ib_qp->context), +- "process tx window query item reset qp failed:%d\n", +- ret); ++ zxdh_dbg(ZXDH_DBG_QP, ++ "process tx window query item reset qp failed:%d\n", ++ ret); + return ZXDH_ERR_RETRY_ACK_ERR; + } + qp->qp_reset_cnt++; +@@ -1932,8 +2125,8 @@ + ib_qp = &iwuqp->vqp.qp; + ret = zxdh_query_qpc(ib_qp, &qpc); + if (ret) { +- verbs_err(verbs_get_ctx(ib_qp->context), +- "process retry ack query qpc failed:%d\n", ret); ++ zxdh_dbg(ZXDH_DBG_QP, "process retry ack query qpc failed:%d\n", ++ ret); + return ZXDH_ERR_RETRY_ACK_ERR; + } + if (!(qpc.retry_cqe_sq_opcode >= ZXDH_RETRY_CQE_SQ_OPCODE_ERR && +@@ -1959,14 +2152,143 @@ + ZXDH_RETRY_CQE_SQ_OPCODE | + ZXDH_TX_READ_RETRY_FLAG_SET); + if (ret) { +- verbs_err(verbs_get_ctx(ib_qp->context), +- "process retry ack modify qpc failed:%d\n", ret); ++ zxdh_dbg(ZXDH_DBG_QP, ++ "process retry ack modify qpc failed:%d\n", ret); + return ZXDH_ERR_RETRY_ACK_ERR; + } + qp->cqe_retry_cnt++; + return ZXDH_ERR_RETRY_ACK_NOT_EXCEED_ERR; + } + ++static enum zxdh_status_code ++zxdh_flush_sq_comp_info(struct zxdh_qp *qp, struct zxdh_cq_poll_info *info, ++ bool *move_cq_head) ++{ ++ enum zxdh_status_code ret; ++ if (!ZXDH_RING_MORE_WORK(qp->sq_ring)) { ++ return ZXDH_ERR_Q_EMPTY; ++ } ++ do { ++ __le64 *sw_wqe; ++ __u64 wqe_qword; ++ __u64 wqe_idx; ++ __u32 quanta; ++ wqe_idx = qp->sq_ring.tail; ++ sw_wqe = qp->sq_base[wqe_idx].elem; ++ get_64bit_val(sw_wqe, 0, &wqe_qword); ++ info->op_type = (__u8)FIELD_GET(ZXDHQPSQ_OPCODE, wqe_qword); ++ quanta = qp->sq_wrtrk_array[wqe_idx].quanta; ++ ret = zxdh_move_ring_tail_quanta(qp, wqe_idx, quanta); ++ if (ret) ++ return ret; ++ ++ if (info->op_type != ZXDH_OP_TYPE_NOP) { ++ info->wr_id = qp->sq_wrtrk_array[wqe_idx].wrid; ++ break; ++ } ++ } while (1); ++ qp->sq_flush_seen = true; ++ if (!ZXDH_RING_MORE_WORK(qp->sq_ring)) { ++ qp->sq_flush_complete = true; ++ } else ++ *move_cq_head = false; ++ return ZXDH_SUCCESS; ++} ++ ++static enum zxdh_status_code zxdh_sq_comp_info(struct zxdh_qp *qp, ++ struct zxdh_cq_poll_info *info, ++ __u32 wqe_idx, __u64 qword0, ++ bool *move_cq_head) ++{ ++ enum zxdh_status_code status_code; ++ switch (info->comp_status) { ++ case ZXDH_COMPL_STATUS_SUCCESS: ++ case ZXDH_COMPL_STATUS_UNKNOWN: ++ break; ++ case ZXDH_COMPL_STATUS_RETRY_ACK_ERR: ++ if (qp->qp_type == ZXDH_QP_TYPE_ROCE_RC) { ++ status_code = process_retry_ack_err(qp, info); ++ return (status_code == ZXDH_ERR_RETRY_ACK_ERR) ? ++ update_cq_poll_info(qp, info, wqe_idx, ++ qword0) : ++ status_code; ++ } ++ break; ++ case ZXDH_COMPL_STATUS_TX_WINDOW_QUERY_ITEM_ERR: ++ if (qp->qp_type == ZXDH_QP_TYPE_ROCE_RC) { ++ status_code = ++ process_tx_window_query_item_err(qp, info); ++ return (status_code == ZXDH_ERR_RETRY_ACK_ERR) ? ++ update_cq_poll_info(qp, info, wqe_idx, ++ qword0) : ++ status_code; ++ } ++ break; ++ case ZXDH_COMPL_STATUS_FLUSHED: ++ return zxdh_flush_sq_comp_info(qp, info, move_cq_head); ++ default: ++ break; ++ } ++ return update_cq_poll_info(qp, info, wqe_idx, qword0); ++} ++ ++static enum zxdh_status_code zxdh_rq_comp_info(struct zxdh_qp *qp, ++ struct zxdh_cq_poll_info *info, ++ __u32 wqe_idx, __u64 qword2, ++ __u64 qword3, bool *move_cq_head) ++{ ++ struct zxdh_uqp *iwuqp = NULL; ++ struct zxdh_usrq *iwusrq = NULL; ++ struct zxdh_srq *srq = NULL; ++ if (qp->is_srq) { ++ if (qp->last_wqe == wqe_idx && unlikely(info->comp_status == ZXDH_COMPL_STATUS_FLUSHED)) ++ return ZXDH_ERR_SRQ_EMPTY; ++ ++ qp->last_wqe = wqe_idx; ++ iwuqp = container_of(qp, struct zxdh_uqp, qp); ++ iwusrq = iwuqp->srq; ++ srq = &iwusrq->srq; ++ zxdh_free_srq_wqe(srq, wqe_idx); ++ info->wr_id = srq->srq_wrid_array[wqe_idx]; ++ zxdh_get_cq_poll_info(qp, info, qword2, qword3); ++ } else { ++ __u32 count; ++ if (unlikely(info->comp_status == ZXDH_COMPL_STATUS_FLUSHED || ++ info->comp_status == ZXDH_COMPL_STATUS_UNKNOWN)) { ++ if (!ZXDH_RING_MORE_WORK(qp->rq_ring)) { ++ return ZXDH_ERR_Q_EMPTY; ++ } ++ wqe_idx = qp->rq_ring.tail; ++ } ++ info->wr_id = qp->rq_wrid_array[wqe_idx]; ++ zxdh_get_cq_poll_info(qp, info, qword2, qword3); ++ /* 计算需要移动的quanta数（考虑卷绕） */ ++ if (wqe_idx >= qp->last_rqe_idx) { ++ count = wqe_idx - qp->last_rqe_idx + 1; ++ } else { ++ /* 卷绕场景 */ ++ count = (qp->rq_ring.size - qp->last_rqe_idx) + wqe_idx + 1; ++ } ++ /* 加锁保护 */ ++ if (unlikely(pthread_spin_lock(qp->quanta_lock))) ++ return ZXDH_ERR_REG_QSET; ++ ZXDH_RING_MOVE_TAIL_BY_COUNT(qp->rq_ring, count); ++ qp->last_rqe_idx = (wqe_idx + 1) % qp->rq_ring.size; ++ if (unlikely(pthread_spin_unlock(qp->quanta_lock))) ++ return ZXDH_ERR_REG_QSET; ++ zxdh_quanta_dbg("[MOVE_TAIL]: wqe_idx=%u qp_num=%u rq_ring.tail=%u last_rqe_idx=%u rq_ring.size=%u count=%u used_quanta_cnt=%u\n", ++ wqe_idx, qp->qp_id, qp->rq_ring.tail, qp->last_rqe_idx, qp->rq_ring.size, count, qp->rq_ring.used_quanta_cnt); ++ if (info->comp_status == ZXDH_COMPL_STATUS_FLUSHED) { ++ qp->rq_flush_seen = true; ++ if (!ZXDH_RING_MORE_WORK(qp->rq_ring)) ++ qp->rq_flush_complete = true; ++ else ++ *move_cq_head = false; ++ } ++ } ++ return ZXDH_SUCCESS; ++} ++ + /** + * zxdh_cq_poll_cmpl - get cq completion info + * @cq: hw cq +@@ -1975,7 +2297,6 @@ + enum zxdh_status_code zxdh_cq_poll_cmpl(struct zxdh_cq *cq, + struct zxdh_cq_poll_info *info) + { +- enum zxdh_status_code status_code; + __u64 comp_ctx, qword0, qword2, qword3; + __le64 *cqe; + struct zxdh_qp *qp; +@@ -1984,9 +2305,6 @@ + int ret_code; + bool move_cq_head = true; + __u8 polarity; +- struct zxdh_usrq *iwusrq = NULL; +- struct zxdh_srq *srq = NULL; +- struct zxdh_uqp *iwuqp; + + cqe = get_current_cqe(cq); + +@@ -2006,7 +2324,7 @@ + ret_code = ZXDH_ERR_Q_DESTROYED; + goto exit; + } +- iwuqp = container_of(qp, struct zxdh_uqp, qp); ++ + info->qp_handle = (zxdh_qp_handle)(unsigned long)qp; + q_type = (__u8)FIELD_GET(ZXDH_CQ_SQ, qword0); + info->solicited_event = (bool)FIELD_GET(ZXDHCQ_SOEVENT, qword0); +@@ -2025,75 +2343,21 @@ + build_comp_status(q_type, info); + + info->qp_id = (__u32)FIELD_GET(ZXDHCQ_QPID, qword2); ++ info->op_type = (__u8)FIELD_GET(ZXDHCQ_OP, qword0); + info->imm_valid = false; +- +- info->qp_handle = (zxdh_qp_handle)(unsigned long)qp; + switch (q_type) { +- case ZXDH_CQE_QTYPE_RQ: +- if (qp->is_srq) { +- iwusrq = iwuqp->srq; +- srq = &iwusrq->srq; +- zxdh_free_srq_wqe(srq, wqe_idx); +- info->wr_id = srq->srq_wrid_array[wqe_idx]; +- zxdh_get_cq_poll_info(qp, info, qword2, qword3); +- } else { +- if (unlikely(info->comp_status == +- ZXDH_COMPL_STATUS_FLUSHED || +- info->comp_status == +- ZXDH_COMPL_STATUS_UNKNOWN)) { +- if (!ZXDH_RING_MORE_WORK(qp->rq_ring)) { +- ret_code = ZXDH_ERR_Q_EMPTY; +- goto exit; +- } +- wqe_idx = qp->rq_ring.tail; +- } +- info->wr_id = qp->rq_wrid_array[wqe_idx]; +- zxdh_get_cq_poll_info(qp, info, qword2, qword3); +- ZXDH_RING_SET_TAIL(qp->rq_ring, wqe_idx + 1); +- if (info->comp_status == ZXDH_COMPL_STATUS_FLUSHED) { +- qp->rq_flush_seen = true; +- if (!ZXDH_RING_MORE_WORK(qp->rq_ring)) +- qp->rq_flush_complete = true; +- else +- move_cq_head = false; +- } +- pring = &qp->rq_ring; +- } +- ret_code = ZXDH_SUCCESS; +- break; + case ZXDH_CQE_QTYPE_SQ: +- if (info->comp_status == ZXDH_COMPL_STATUS_RETRY_ACK_ERR && +- qp->qp_type == ZXDH_QP_TYPE_ROCE_RC) { +- status_code = process_retry_ack_err(qp, info); +- if (status_code == ZXDH_ERR_RETRY_ACK_ERR) { +- update_cq_poll_info(qp, info, wqe_idx, qword0); +- ret_code = ZXDH_SUCCESS; +- } else { +- ret_code = status_code; +- } +- } else if (info->comp_status == +- ZXDH_COMPL_STATUS_TX_WINDOW_QUERY_ITEM_ERR && +- qp->qp_type == ZXDH_QP_TYPE_ROCE_RC) { +- status_code = +- process_tx_window_query_item_err(qp, info); +- if (status_code == ZXDH_ERR_RETRY_ACK_ERR) { +- update_cq_poll_info(qp, info, wqe_idx, qword0); +- ret_code = ZXDH_SUCCESS; +- } else { +- ret_code = status_code; +- } +- } else if (info->comp_status == ZXDH_COMPL_STATUS_FLUSHED) { +- info->wr_id = qp->sq_wrtrk_array[wqe_idx].wrid; +- ZXDH_RING_INIT(qp->sq_ring, qp->sq_ring.size); +- ret_code = ZXDH_SUCCESS; +- } else { +- update_cq_poll_info(qp, info, wqe_idx, qword0); +- ret_code = ZXDH_SUCCESS; +- } ++ ret_code = zxdh_sq_comp_info(qp, info, wqe_idx, qword0, ++ &move_cq_head); ++ pring = &qp->sq_ring; ++ break; ++ case ZXDH_CQE_QTYPE_RQ: ++ ret_code = zxdh_rq_comp_info(qp, info, wqe_idx, qword2, qword3, ++ &move_cq_head); ++ pring = &qp->rq_ring; + break; + default: +- zxdh_dbg(verbs_get_ctx(iwuqp->vqp.qp.context), ZXDH_DBG_CQ, +- "zxdh get cqe type unknow!\n"); ++ zxdh_dbg(ZXDH_DBG_CQ, "zxdh get cqe type unknow!\n"); + ret_code = ZXDH_ERR_Q_DESTROYED; + break; + } +@@ -2129,6 +2393,9 @@ + { + int scount = 1; + ++ if (wqdepth == 0) ++ return 0; ++ + for (wqdepth--; scount <= 16; scount *= 2) + wqdepth |= wqdepth >> scount; + +@@ -2143,6 +2410,9 @@ + { + int scount = 1; + ++ if (wqdepth == 0) ++ return 0; ++ + for (wqdepth--; scount <= 16; scount *= 2) + wqdepth |= wqdepth >> scount; + +@@ -2210,6 +2480,26 @@ + __u32 sq_size, __u8 shift, + __u32 *sqdepth) + { ++ __u32 sq_quanta; ++ if (sq_size > ZXDH_MAX_SQ_DEPTH) ++ return ZXDH_ERR_INVALID_SIZE; ++ ++ sq_quanta = sq_size << shift; ++ /* 标准verbs路径：直接不使用RSVD */ ++ *sqdepth = zxdh_qp_round_up(sq_quanta); ++ ++ if (*sqdepth < (ZXDH_QP_SW_MIN_WQSIZE << shift)) ++ *sqdepth = ZXDH_QP_SW_MIN_WQSIZE << shift; ++ else if (*sqdepth > dev_attrs->max_hw_wq_quanta) ++ return ZXDH_ERR_INVALID_SIZE; ++ ++ return 0; ++} ++ ++enum zxdh_status_code zxdh_devx_get_sqdepth(struct zxdh_dev_attrs *dev_attrs, ++ __u32 sq_size, __u8 shift, ++ __u32 *sqdepth) ++{ + if (sq_size > ZXDH_MAX_SQ_DEPTH) + return ZXDH_ERR_INVALID_SIZE; + +@@ -2234,10 +2524,25 @@ + __u32 rq_size, __u8 shift, + __u32 *rqdepth) + { +- *rqdepth = zxdh_qp_round_up((rq_size << shift) + ZXDH_RQ_RSVD); ++ /* 标准verbs路径：直接不使用RSVD */ ++ *rqdepth = zxdh_qp_round_up(rq_size); ++ ++ if (*rqdepth < ZXDH_QP_SW_MIN_WQSIZE) ++ *rqdepth = ZXDH_QP_SW_MIN_WQSIZE; ++ else if (*rqdepth > dev_attrs->max_hw_rq_quanta) ++ return ZXDH_ERR_INVALID_SIZE; ++ ++ return 0; ++} ++ ++enum zxdh_status_code zxdh_devx_get_rqdepth(struct zxdh_dev_attrs *dev_attrs, ++ __u32 rq_size, __u8 shift, ++ __u32 *rqdepth) ++{ ++ *rqdepth = zxdh_qp_round_up(rq_size + ZXDH_RQ_RSVD); + +- if (*rqdepth < (ZXDH_QP_SW_MIN_WQSIZE << shift)) +- *rqdepth = ZXDH_QP_SW_MIN_WQSIZE << shift; ++ if (*rqdepth < ZXDH_QP_SW_MIN_WQSIZE) ++ *rqdepth = ZXDH_QP_SW_MIN_WQSIZE; + else if (*rqdepth > dev_attrs->max_hw_rq_quanta) + return ZXDH_ERR_INVALID_SIZE; + +@@ -2306,7 +2611,14 @@ + qp->rq_wqe_size = rqshift; + ZXDH_RING_INIT(qp->rq_ring, qp->rq_size); + qp->rq_wqe_size_multiplier = 1 << rqshift; ++ /* 新增：初始化last_sqe_idx和last_rqe_idx */ ++ qp->last_sqe_idx = 0; ++ qp->last_rqe_idx = 0; + qp->wqe_ops = iw_wqe_ops; ++ qp->last_wqe = 0; ++ if (qp->is_ext_qp) { ++ qp->wqe_alloc_db = info->ext_sq_db; ++ } + return ret_code; + } + +@@ -2340,10 +2652,13 @@ + void zxdh_clean_cq(void *q, struct zxdh_cq *cq) + { + __le64 *cqe; +- __u64 qword3, comp_ctx; ++ __u64 qword0, comp_ctx; + __u32 cq_head; + __u8 polarity, temp; + ++ if (!cq || cq->cq_ring.size == 0) ++ return; ++ + cq_head = cq->cq_ring.head; + temp = cq->polarity; + do { +@@ -2353,8 +2668,8 @@ + .buf; + else + cqe = cq->cq_base[cq_head].buf; +- get_64bit_val(cqe, 24, &qword3); +- polarity = (__u8)FIELD_GET(ZXDH_CQ_VALID, qword3); ++ get_64bit_val(cqe, 0, &qword0); ++ polarity = (__u8)FIELD_GET(ZXDH_CQ_VALID, qword0); + + if (polarity != temp) + break; +@@ -2391,8 +2706,6 @@ + if (!wqe) + return ZXDH_ERR_QP_TOOMANY_WRS_POSTED; + +- zxdh_clr_wqes(qp, wqe_idx); +- + set_64bit_val(wqe, 0, 0); + set_64bit_val(wqe, 8, 0); + set_64bit_val(wqe, 16, 0); +@@ -2404,6 +2717,7 @@ + udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */ + + set_64bit_val(wqe, 24, hdr); ++ read_wqe_ctrl_hdr_coherent_if_need(qp, wqe); + if (post_sq) + zxdh_qp_post_wr(qp); + +@@ -2411,19 +2725,6 @@ + } + + /** +- * zxdh_fragcnt_to_quanta_sq - calculate quanta based on fragment count for SQ +- * @frag_cnt: number of fragments +- * @quanta: quanta for frag_cnt +- */ +-enum zxdh_status_code zxdh_fragcnt_to_quanta_sq(__u32 frag_cnt, __u16 *quanta) +-{ +- if (frag_cnt > ZXDH_MAX_SQ_FRAG) +- return ZXDH_ERR_INVALID_FRAG_COUNT; +- *quanta = frag_cnt / 2 + 1; +- return 0; +-} +- +-/** + * zxdh_fragcnt_to_wqesize_rq - calculate wqe size based on fragment count for RQ + * @frag_cnt: number of fragments + * @wqe_size: size in bytes given frag_cnt +@@ -2500,11 +2801,11 @@ + int zxdh_get_srqdepth(__u32 max_hw_srq_quanta, __u32 srq_size, __u8 shift, + __u32 *srqdepth) + { +- *srqdepth = zxdh_qp_round_up((srq_size << shift) + ZXDH_SRQ_RSVD); ++ *srqdepth = zxdh_qp_round_up(srq_size + ZXDH_SRQ_RSVD); + +- if (*srqdepth < (ZXDH_QP_SW_MIN_WQSIZE << shift)) +- *srqdepth = ZXDH_QP_SW_MIN_WQSIZE << shift; +- else if ((*srqdepth >> shift) > max_hw_srq_quanta) ++ if (*srqdepth < ZXDH_QP_SW_MIN_WQSIZE) ++ *srqdepth = ZXDH_QP_SW_MIN_WQSIZE; ++ else if (*srqdepth > max_hw_srq_quanta) + return ZXDH_ERR_INVALID_SIZE; + + return 0; +@@ -2573,6 +2874,16 @@ + ZXDH_RING_INIT(srq->srq_list_ring, srq->srq_list_size); + srq->srq_ring.tail = srq->srq_size - 1; + srq->srq_list_polarity = 1; ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s srq_wqe_size_multiplier:%d srqshift:%d\n", ++ __func__, srq->srq_wqe_size_multiplier, srqshift); ++ zxdh_dbg( ++ ZXDH_DBG_SRQ, ++ "%s srq->srq_id:%d srq_base:0x%p srq_list_base:0x%p srq_db_base:0x%p\n", ++ __func__, srq->srq_id, srq->srq_base, srq->srq_list_base, ++ srq->srq_db_base); ++ zxdh_dbg(ZXDH_DBG_SRQ, ++ "%s srq->srq_id:%d srq_ring_size:%d srq->srq_list_size:%d\n", ++ __func__, srq->srq_id, srq_ring_size, srq->srq_list_size); + return 0; + } + +@@ -2593,4 +2904,6 @@ + set_64bit_val(wqe, 0, hdr); + + pthread_spin_unlock(&iwusrq->lock); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s srq->srq_id:%d wqe_index:%d\n", __func__, ++ srq->srq_id, wqe_index); + } +diff -ruN baseline/providers/zrdma/zxdh_status.h b/providers/zrdma/zxdh_status.h +--- a/providers/zrdma/zxdh_status.h 2026-08-06 15:37:55.353862831 +0800 ++++ b/providers/zrdma/zxdh_status.h 2026-08-06 15:29:37.791713076 +0800 +@@ -3,6 +3,8 @@ + #ifndef ZXDH_STATUS_H + #define ZXDH_STATUS_H + ++#include ++ + /* Error Codes */ + enum zxdh_status_code { + ZXDH_SUCCESS = 0, +@@ -71,5 +73,242 @@ + ZXDH_ERR_INVALID_FRAG_LEN = -75, + ZXDH_ERR_RETRY_ACK_ERR = -76, + ZXDH_ERR_RETRY_ACK_NOT_EXCEED_ERR = -77, ++ ZXDH_ERR_BIND_MW_TYPE = -78, ++ ZXDH_ERR_INVALID_WR_OP_TYPE = -79, ++ ZXDH_ERR_BAD_QP = -80, ++ ZXDH_ERR_SRQ_EMPTY = -81, + }; ++ ++static inline int zxdh_status_code_to_errno(enum zxdh_status_code status) ++{ ++ switch (status) { ++ case ZXDH_SUCCESS: ++ return 0; ++ case ZXDH_ERR_PARAM: ++ case ZXDH_ERR_INVALID_PD_ID: ++ case ZXDH_ERR_INVALID_QP_ID: ++ case ZXDH_ERR_INVALID_CQ_ID: ++ case ZXDH_ERR_INVALID_CEQ_ID: ++ case ZXDH_ERR_INVALID_AEQ_ID: ++ case ZXDH_ERR_INVALID_SIZE: ++ case ZXDH_ERR_INVALID_ARP_INDEX: ++ case ZXDH_ERR_INVALID_FPM_FUNC_ID: ++ case ZXDH_ERR_QP_INVALID_MSG_SIZE: ++ case ZXDH_ERR_INVALID_FRAG_COUNT: ++ case ZXDH_ERR_INVALID_ALIGNMENT: ++ case ZXDH_ERR_INVALID_PUSH_PAGE_INDEX: ++ case ZXDH_ERR_INVALID_VF_ID: ++ case ZXDH_ERR_INVALID_HMCFN_ID: ++ case ZXDH_ERR_INVALID_PBLE_INDEX: ++ case ZXDH_ERR_INVALID_SD_INDEX: ++ case ZXDH_ERR_INVALID_PAGE_DESC_INDEX: ++ case ZXDH_ERR_INVALID_SD_TYPE: ++ case ZXDH_ERR_INVALID_HMC_OBJ_INDEX: ++ case ZXDH_ERR_INVALID_HMC_OBJ_COUNT: ++ case ZXDH_ERR_INVALID_FEAT_CNT: ++ case ZXDH_ERR_INVALID_FRAG_LEN: ++ case ZXDH_ERR_INVALID_MAC_ADDR: ++ case ZXDH_ERR_OPCODE_MISMATCH: ++ case ZXDH_ERR_INVALID_INLINE_DATA_SIZE: ++ case ZXDH_ERR_BIND_MW_TYPE: ++ case ZXDH_ERR_INVALID_WR_OP_TYPE: ++ case ZXDH_ERR_BAD_QP: ++ return EINVAL; ++ case ZXDH_ERR_NO_MEMORY: ++ case ZXDH_ERR_NO_PBLCHUNKS_AVAILABLE: ++ case ZXDH_ERR_REG_CQ_FULL: ++ case ZXDH_ERR_RING_FULL: ++ case ZXDH_ERR_QP_TOOMANY_WRS_POSTED: ++ return ENOMEM; ++ case ZXDH_ERR_Q_EMPTY: ++ case ZXDH_ERR_FLUSHED_Q: ++ case ZXDH_ERR_Q_DESTROYED: ++ return ESHUTDOWN; ++ case ZXDH_ERR_TIMEOUT: ++ return ETIMEDOUT; ++ case ZXDH_ERR_NVM: ++ case ZXDH_ERR_NVM_CHECKSUM: ++ case ZXDH_ERR_CFG: ++ case ZXDH_ERR_RESET_FAILED: ++ case ZXDH_ERR_SWFW_SYNC: ++ case ZXDH_ERR_CQP_COMPL_ERROR: ++ case ZXDH_ERR_BACKING_PAGE_ERROR: ++ case ZXDH_ERR_BAD_IWARP_CQE: ++ case ZXDH_ERR_NVM_BLANK_MODE: ++ case ZXDH_ERR_PE_DOORBELL_NOT_ENA: ++ case ZXDH_ERR_FIRMWARE_API_VER: ++ case ZXDH_ERR_CQ_COMPL_ERROR: ++ case ZXDH_ERR_VF_MSG_ERROR: ++ case ZXDH_ERR_REG_QSET: ++ case ZXDH_ERR_FEATURES_OP: ++ case ZXDH_ERR_RETRY_ACK_ERR: ++ case ZXDH_ERR_RETRY_ACK_NOT_EXCEED_ERR: ++ return EIO; ++ case ZXDH_ERR_MPA_CRC: ++ case ZXDH_ERR_SEQ_NUM: ++ case ZXDH_ERR_BAD_STAG: ++ return EPROTO; ++ case ZXDH_ERR_DEVICE_NOT_SUPPORTED: ++ case ZXDH_ERR_NOT_IMPL: ++ case ZXDH_NOT_SUPPORTED: ++ return ENOTSUP; ++ case ZXDH_ERR_NOT_READY: ++ return EBUSY; ++ case ZXDH_ERR_BUF_TOO_SHORT: ++ return EMSGSIZE; ++ case ZXDH_ERR_BAD_PTR: ++ case ZXDH_ERR_MEMCPY_FAILED: ++ case ZXDH_ERR_LIST_EMPTY: ++ case ZXDH_ERR_NO_TXBUFS: ++ case ZXDH_ERR_NO_INTR: ++ default: ++ return EINVAL; ++ } ++} ++ ++static inline const char *zxdh_status_code_to_string(enum zxdh_status_code status) ++{ ++ switch (status) { ++ case ZXDH_SUCCESS: ++ return "ZXDH_SUCCESS"; ++ case ZXDH_ERR_NVM: ++ return "ZXDH_ERR_NVM"; ++ case ZXDH_ERR_NVM_CHECKSUM: ++ return "ZXDH_ERR_NVM_CHECKSUM"; ++ case ZXDH_ERR_CFG: ++ return "ZXDH_ERR_CFG"; ++ case ZXDH_ERR_PARAM: ++ return "ZXDH_ERR_PARAM"; ++ case ZXDH_ERR_DEVICE_NOT_SUPPORTED: ++ return "ZXDH_ERR_DEVICE_NOT_SUPPORTED"; ++ case ZXDH_ERR_RESET_FAILED: ++ return "ZXDH_ERR_RESET_FAILED"; ++ case ZXDH_ERR_SWFW_SYNC: ++ return "ZXDH_ERR_SWFW_SYNC"; ++ case ZXDH_ERR_NO_MEMORY: ++ return "ZXDH_ERR_NO_MEMORY"; ++ case ZXDH_ERR_BAD_PTR: ++ return "ZXDH_ERR_BAD_PTR"; ++ case ZXDH_ERR_INVALID_PD_ID: ++ return "ZXDH_ERR_INVALID_PD_ID"; ++ case ZXDH_ERR_INVALID_QP_ID: ++ return "ZXDH_ERR_INVALID_QP_ID"; ++ case ZXDH_ERR_INVALID_CQ_ID: ++ return "ZXDH_ERR_INVALID_CQ_ID"; ++ case ZXDH_ERR_INVALID_CEQ_ID: ++ return "ZXDH_ERR_INVALID_CEQ_ID"; ++ case ZXDH_ERR_INVALID_AEQ_ID: ++ return "ZXDH_ERR_INVALID_AEQ_ID"; ++ case ZXDH_ERR_INVALID_SIZE: ++ return "ZXDH_ERR_INVALID_SIZE"; ++ case ZXDH_ERR_INVALID_ARP_INDEX: ++ return "ZXDH_ERR_INVALID_ARP_INDEX"; ++ case ZXDH_ERR_INVALID_FPM_FUNC_ID: ++ return "ZXDH_ERR_INVALID_FPM_FUNC_ID"; ++ case ZXDH_ERR_QP_INVALID_MSG_SIZE: ++ return "ZXDH_ERR_QP_INVALID_MSG_SIZE"; ++ case ZXDH_ERR_QP_TOOMANY_WRS_POSTED: ++ return "ZXDH_ERR_QP_TOOMANY_WRS_POSTED"; ++ case ZXDH_ERR_INVALID_FRAG_COUNT: ++ return "ZXDH_ERR_INVALID_FRAG_COUNT"; ++ case ZXDH_ERR_Q_EMPTY: ++ return "ZXDH_ERR_Q_EMPTY"; ++ case ZXDH_ERR_INVALID_ALIGNMENT: ++ return "ZXDH_ERR_INVALID_ALIGNMENT"; ++ case ZXDH_ERR_FLUSHED_Q: ++ return "ZXDH_ERR_FLUSHED_Q"; ++ case ZXDH_ERR_INVALID_PUSH_PAGE_INDEX: ++ return "ZXDH_ERR_INVALID_PUSH_PAGE_INDEX"; ++ case ZXDH_ERR_INVALID_INLINE_DATA_SIZE: ++ return "ZXDH_ERR_INVALID_INLINE_DATA_SIZE"; ++ case ZXDH_ERR_TIMEOUT: ++ return "ZXDH_ERR_TIMEOUT"; ++ case ZXDH_ERR_OPCODE_MISMATCH: ++ return "ZXDH_ERR_OPCODE_MISMATCH"; ++ case ZXDH_ERR_CQP_COMPL_ERROR: ++ return "ZXDH_ERR_CQP_COMPL_ERROR"; ++ case ZXDH_ERR_INVALID_VF_ID: ++ return "ZXDH_ERR_INVALID_VF_ID"; ++ case ZXDH_ERR_INVALID_HMCFN_ID: ++ return "ZXDH_ERR_INVALID_HMCFN_ID"; ++ case ZXDH_ERR_BACKING_PAGE_ERROR: ++ return "ZXDH_ERR_BACKING_PAGE_ERROR"; ++ case ZXDH_ERR_NO_PBLCHUNKS_AVAILABLE: ++ return "ZXDH_ERR_NO_PBLCHUNKS_AVAILABLE"; ++ case ZXDH_ERR_INVALID_PBLE_INDEX: ++ return "ZXDH_ERR_INVALID_PBLE_INDEX"; ++ case ZXDH_ERR_INVALID_SD_INDEX: ++ return "ZXDH_ERR_INVALID_SD_INDEX"; ++ case ZXDH_ERR_INVALID_PAGE_DESC_INDEX: ++ return "ZXDH_ERR_INVALID_PAGE_DESC_INDEX"; ++ case ZXDH_ERR_INVALID_SD_TYPE: ++ return "ZXDH_ERR_INVALID_SD_TYPE"; ++ case ZXDH_ERR_MEMCPY_FAILED: ++ return "ZXDH_ERR_MEMCPY_FAILED"; ++ case ZXDH_ERR_INVALID_HMC_OBJ_INDEX: ++ return "ZXDH_ERR_INVALID_HMC_OBJ_INDEX"; ++ case ZXDH_ERR_INVALID_HMC_OBJ_COUNT: ++ return "ZXDH_ERR_INVALID_HMC_OBJ_COUNT"; ++ case ZXDH_ERR_BUF_TOO_SHORT: ++ return "ZXDH_ERR_BUF_TOO_SHORT"; ++ case ZXDH_ERR_BAD_IWARP_CQE: ++ return "ZXDH_ERR_BAD_IWARP_CQE"; ++ case ZXDH_ERR_NVM_BLANK_MODE: ++ return "ZXDH_ERR_NVM_BLANK_MODE"; ++ case ZXDH_ERR_NOT_IMPL: ++ return "ZXDH_ERR_NOT_IMPL"; ++ case ZXDH_ERR_PE_DOORBELL_NOT_ENA: ++ return "ZXDH_ERR_PE_DOORBELL_NOT_ENA"; ++ case ZXDH_ERR_NOT_READY: ++ return "ZXDH_ERR_NOT_READY"; ++ case ZXDH_NOT_SUPPORTED: ++ return "ZXDH_NOT_SUPPORTED"; ++ case ZXDH_ERR_FIRMWARE_API_VER: ++ return "ZXDH_ERR_FIRMWARE_API_VER"; ++ case ZXDH_ERR_RING_FULL: ++ return "ZXDH_ERR_RING_FULL"; ++ case ZXDH_ERR_MPA_CRC: ++ return "ZXDH_ERR_MPA_CRC"; ++ case ZXDH_ERR_NO_TXBUFS: ++ return "ZXDH_ERR_NO_TXBUFS"; ++ case ZXDH_ERR_SEQ_NUM: ++ return "ZXDH_ERR_SEQ_NUM"; ++ case ZXDH_ERR_LIST_EMPTY: ++ return "ZXDH_ERR_LIST_EMPTY"; ++ case ZXDH_ERR_INVALID_MAC_ADDR: ++ return "ZXDH_ERR_INVALID_MAC_ADDR"; ++ case ZXDH_ERR_BAD_STAG: ++ return "ZXDH_ERR_BAD_STAG"; ++ case ZXDH_ERR_CQ_COMPL_ERROR: ++ return "ZXDH_ERR_CQ_COMPL_ERROR"; ++ case ZXDH_ERR_Q_DESTROYED: ++ return "ZXDH_ERR_Q_DESTROYED"; ++ case ZXDH_ERR_INVALID_FEAT_CNT: ++ return "ZXDH_ERR_INVALID_FEAT_CNT"; ++ case ZXDH_ERR_REG_CQ_FULL: ++ return "ZXDH_ERR_REG_CQ_FULL"; ++ case ZXDH_ERR_VF_MSG_ERROR: ++ return "ZXDH_ERR_VF_MSG_ERROR"; ++ case ZXDH_ERR_NO_INTR: ++ return "ZXDH_ERR_NO_INTR"; ++ case ZXDH_ERR_REG_QSET: ++ return "ZXDH_ERR_REG_QSET"; ++ case ZXDH_ERR_FEATURES_OP: ++ return "ZXDH_ERR_FEATURES_OP"; ++ case ZXDH_ERR_INVALID_FRAG_LEN: ++ return "ZXDH_ERR_INVALID_FRAG_LEN"; ++ case ZXDH_ERR_RETRY_ACK_ERR: ++ return "ZXDH_ERR_RETRY_ACK_ERR"; ++ case ZXDH_ERR_RETRY_ACK_NOT_EXCEED_ERR: ++ return "ZXDH_ERR_RETRY_ACK_NOT_EXCEED_ERR"; ++ case ZXDH_ERR_BIND_MW_TYPE: ++ return "ZXDH_ERR_BIND_MW_TYPE"; ++ case ZXDH_ERR_INVALID_WR_OP_TYPE: ++ return "ZXDH_ERR_INVALID_WR_OP_TYPE"; ++ case ZXDH_ERR_BAD_QP: ++ return "ZXDH_ERR_BAD_QP"; ++ default: ++ return "UNKNOWN_STATUS"; ++ } ++} + #endif /* ZXDH_STATUS_H */ +diff -ruN baseline/providers/zrdma/zxdh_verbs.c b/providers/zrdma/zxdh_verbs.c +--- a/providers/zrdma/zxdh_verbs.c 2026-08-06 15:37:55.359863242 +0800 ++++ b/providers/zrdma/zxdh_verbs.c 2026-08-06 15:29:37.015659780 +0800 +@@ -17,21 +17,13 @@ + #include + #include + +-#include "main.h" ++#include "zxdh_zrdma.h" + #include "zxdh_abi.h" ++#include "wqe_rate_limit.h" ++#include "zxdh_dv.h" ++extern unsigned int rate_limit_flag[MAX_VHCA_NUM]; + + uint32_t zxdh_debug_mask; +- +-static const unsigned int zxdh_roce_mtu[] = { +- [IBV_MTU_256] = 256, [IBV_MTU_512] = 512, [IBV_MTU_1024] = 1024, +- [IBV_MTU_2048] = 2048, [IBV_MTU_4096] = 4096, +-}; +- +-static inline unsigned int mtu_enum_to_int(enum ibv_mtu mtu) +-{ +- return zxdh_roce_mtu[mtu]; +-} +- + static inline void print_fw_ver(uint64_t fw_ver, char *str, size_t len) + { + uint16_t major, minor, sub_minor, sub_major; +@@ -59,7 +51,7 @@ + while (num < ib_wr->num_sge) { + *len += ib_wr->sg_list[num].length; + if (*len > ZXDH_MAX_INLINE_DATA_SIZE) { +- return -EINVAL; ++ return ZXDH_ERR_INVALID_INLINE_DATA_SIZE; + } + memcpy(inline_data + offset, + (void *)(uintptr_t)ib_wr->sg_list[num].addr, +@@ -84,7 +76,27 @@ + struct ib_uverbs_ex_query_device_resp resp = {}; + size_t resp_size = sizeof(resp); + int ret; +- ++#if IBVERBS_PABI_VERSION <= 21 ++ struct ibv_query_device_ex cmd = {}; ++ uint64_t fw_ver; ++ ret = ibv_cmd_query_device_ex(context, input, attr, attr_size, ++ (uint64_t *)&fw_ver, &cmd, sizeof(cmd), ++ sizeof(cmd), &resp, resp_size, resp_size); ++ if (ret) ++ return ret; ++ print_fw_ver(fw_ver, attr->orig_attr.fw_ver, ++ sizeof(attr->orig_attr.fw_ver)); ++#elif IBVERBS_PABI_VERSION == 25 || IBVERBS_PABI_VERSION == 22 ++ struct ibv_query_device_ex cmd = {}; ++ uint64_t fw_ver; ++ ret = ibv_cmd_query_device_ex(context, input, attr, attr_size, ++ (uint64_t *)&fw_ver, &cmd, sizeof(cmd), ++ &resp, resp_size); ++ if (ret) ++ return ret; ++ print_fw_ver(fw_ver, attr->orig_attr.fw_ver, ++ sizeof(attr->orig_attr.fw_ver)); ++#else + ret = ibv_cmd_query_device_any(context, input, attr, attr_size, &resp, + &resp_size); + if (ret) +@@ -92,9 +104,27 @@ + + print_fw_ver(resp.base.fw_ver, attr->orig_attr.fw_ver, + sizeof(attr->orig_attr.fw_ver)); ++#endif ++ return 0; ++} + ++#if IBVERBS_PABI_VERSION == 25 ++int zxdh_uquery_device(struct ibv_context *context, ++ struct ibv_device_attr *dev_attr) ++{ ++ struct ibv_query_device cmd; ++ uint64_t fw_ver; ++ int status; ++ ++ memset(dev_attr, 0, sizeof(struct ibv_device_attr)); ++ status = ibv_cmd_query_device(context, dev_attr, &fw_ver, &cmd, ++ sizeof(cmd)); ++ if (status) ++ return status; ++ print_fw_ver(fw_ver, dev_attr->fw_ver, sizeof(dev_attr->fw_ver)); + return 0; + } ++#endif + + /** + * zxdh_uquery_port - get port attributes (msg size, lnk, mtu...) +@@ -167,8 +197,8 @@ + * @hca_va: hca_va + * @access: access allowed on this mr + */ +-struct ibv_mr *zxdh_ureg_mr(struct ibv_pd *pd, void *addr, size_t length, +- uint64_t hca_va, int access) ++ struct ibv_mr *zxdh_ureg_mr(struct ibv_pd *pd, void *addr, size_t length, ++ uint64_t hca_va, int access) + { + struct zxdh_umr *umr; + struct zxdh_ureg_mr cmd; +@@ -197,6 +227,7 @@ + return &umr->vmr.ibv_mr; + } + ++ + /* + * zxdh_urereg_mr - re-register memory region + * @vmr: mr that was allocated +@@ -306,7 +337,6 @@ + int zxdh_udealloc_mw(struct ibv_mw *mw) + { + int ret; +- + ret = ibv_cmd_dealloc_mw(mw); + if (ret) + return ret; +@@ -315,15 +345,135 @@ + return 0; + } + +-static void *zxdh_alloc_hw_buf(size_t size) ++int zxdh_rdmanl_saw_err_cb(struct sockaddr_nl *nla, struct nlmsgerr *nlerr, ++ void *arg) + { +- void *buf; ++ bool *failed = arg; ++ ++ *failed = true; ++ return 0; ++} ++ ++struct nl_sock *zxdh_rdmanl_socket_alloc(void) ++{ ++ struct nl_sock *nl; ++ ++ nl = nl_socket_alloc(); ++ if (!nl) ++ return NULL; ++ nl_socket_disable_auto_ack(nl); ++ nl_socket_disable_msg_peek(nl); ++ ++ if (nl_connect(nl, NETLINK_RDMA)) { ++ nl_socket_free(nl); ++ return NULL; ++ } ++ return nl; ++} ++int zxdh_rdmanl_get_copy_on_fork(struct nl_sock *nl, ++ nl_recvmsg_msg_cb_t cb_func, void *data) ++{ ++ bool failed = false; ++ int ret; ++ ++ if (nl_send_simple( ++ nl, RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_SYS_GET), ++ 0, NULL, 0) < 0) ++ return -1; ++ ++ if (nl_socket_modify_err_cb(nl, NL_CB_CUSTOM, zxdh_rdmanl_saw_err_cb, ++ &failed)) ++ return -1; ++ if (nl_socket_modify_cb(nl, NL_CB_VALID, NL_CB_CUSTOM, cb_func, data)) ++ return -1; ++ do { ++ ret = nl_recvmsgs_default(nl); ++ } while (ret > 0); ++ nl_socket_modify_err_cb(nl, NL_CB_CUSTOM, NULL, NULL); ++ ++ if (ret || failed) ++ return -1; ++ return 0; ++} ++ ++#define ZXDH_RDMA_NLDEV_SYS_ATTR_COPY_ON_FORK 93 ++#define ZXDH_RDMA_NLDEV_ATTR_MAX 99 ++ ++struct nla_policy zxdh_rdmanl_policy[ZXDH_RDMA_NLDEV_ATTR_MAX] = { ++ [ZXDH_RDMA_NLDEV_SYS_ATTR_COPY_ON_FORK] = { .type = NLA_U8 }, ++}; ++ ++int zxdh_get_copy_on_fork_cb(struct nl_msg *msg, void *data) ++{ ++ struct nlattr *tb[ZXDH_RDMA_NLDEV_ATTR_MAX]; ++ int ret; ++ ++ ret = nlmsg_parse(nlmsg_hdr(msg), 0, tb, ZXDH_RDMA_NLDEV_ATTR_MAX - 1, ++ zxdh_rdmanl_policy); ++ if (ret < 0) ++ return ret; ++ ++ /* Older kernels don't support COF and don't report it through nl */ ++ if (!tb[ZXDH_RDMA_NLDEV_SYS_ATTR_COPY_ON_FORK]) { ++ *(uint8_t *)data = 0; ++ return NL_OK; ++ } ++ ++ *(uint8_t *)data = ++ nla_get_u8(tb[ZXDH_RDMA_NLDEV_SYS_ATTR_COPY_ON_FORK]); ++ return NL_OK; ++} ++ ++bool zxdh_get_copy_on_fork(void) ++{ ++ struct nl_sock *nl; ++ uint8_t cof; ++ ++ nl = zxdh_rdmanl_socket_alloc(); ++ if (!nl) ++ return false; ++ ++ if (zxdh_rdmanl_get_copy_on_fork(nl, zxdh_get_copy_on_fork_cb, &cof)) ++ cof = false; ++ ++ nl_socket_free(nl); ++ return cof; ++} ++ ++bool zxdh_get_fork_safety(void) ++{ ++ static int is_fork_safed = 0xff; ++ if (is_fork_safed == 0xff) { ++ is_fork_safed = zxdh_get_copy_on_fork(); ++ } ++ return is_fork_safed != 0; ++} ++ ++static inline unsigned long zxdh_align_up(unsigned long val, unsigned long align_val) ++{ ++ return (val + align_val - 1) & ~(align_val - 1); ++} + +- buf = memalign(ZXDH_HW_PAGE_SIZE, size); ++void *zxdh_alloc_hw_buf(size_t size) ++{ ++ void *buf; ++ size_t al_size; + ++ __u64 page_size = ++ (ZXDH_SYS_PAGE_SIZE == ZXDH_HW_64K_PAGE_SIZE) ? ++ ZXDH_HW_64K_PAGE_SIZE : ++ ZXDH_HW_PAGE_SIZE; ++ al_size = zxdh_align_up((unsigned long)size, (unsigned long)page_size); ++ buf = memalign(page_size, al_size); + if (!buf) + return NULL; +- if (ibv_dontfork_range(buf, size)) { ++ ++ /* 禁用透明巨页以避免内存碎页和性能问题 */ ++ madvise(buf, al_size, MADV_NOHUGEPAGE); ++ ++ if (zxdh_get_fork_safety()) ++ return buf; ++ if (ibv_dontfork_range(buf, al_size)) { + free(buf); + return NULL; + } +@@ -333,7 +483,14 @@ + + static void zxdh_free_hw_buf(void *buf, size_t size) + { +- ibv_dofork_range(buf, size); ++ size_t al_size; ++ ++ __u64 page_size = ++ (ZXDH_SYS_PAGE_SIZE == ZXDH_HW_64K_PAGE_SIZE) ? ++ ZXDH_HW_64K_PAGE_SIZE : ++ ZXDH_HW_PAGE_SIZE; ++ al_size = zxdh_align_up((unsigned long)size, (unsigned long)page_size); ++ ibv_dofork_range(buf, al_size); + free(buf); + } + +@@ -343,12 +500,8 @@ + */ + static inline int get_cq_size(int ncqe) + { +- ncqe++; +- +- /* Completions with immediate require 1 extra entry */ + if (ncqe < ZXDH_U_MINCQ_SIZE) + ncqe = ZXDH_U_MINCQ_SIZE; +- + return ncqe; + } + +@@ -380,6 +533,8 @@ + size_t total_size; + __u32 cq_pages; + int ret, ncqe; ++ __u64 resize_supported; ++ __u64 addr_offset = 0; + + iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); + dev_attrs = &iwvctx->dev_attrs; +@@ -390,6 +545,14 @@ + return NULL; + } + ++ info.cq_size = get_cq_size(attr_ex->cqe); ++ info.cq_size = zxdh_cq_round_up(info.cq_size); ++ if (info.cq_size > dev_attrs->max_hw_cq_size) { ++ errno = EINVAL; ++ return NULL; ++ } ++ if (info.cq_size * 2 <= dev_attrs->max_hw_cq_size) ++ info.cq_size = info.cq_size * 2; + /* save the cqe requested by application */ + ncqe = attr_ex->cqe; + iwucq = calloc(1, sizeof(*iwucq)); +@@ -404,19 +567,18 @@ + } + + iwucq->resize_enable = false; +- info.cq_size = get_cq_size(attr_ex->cqe); +- info.cq_size = zxdh_cq_round_up(info.cq_size); + iwucq->comp_vector = attr_ex->comp_vector; + list_head_init(&iwucq->resize_list); + total_size = get_cq_total_bytes(info.cq_size); + cq_pages = total_size >> ZXDH_HW_PAGE_SHIFT; ++ resize_supported = dev_attrs->feature_flags & ZXDH_FEATURE_CQ_RESIZE; + +- if (!(dev_attrs->feature_flags & ZXDH_FEATURE_CQ_RESIZE)) ++ if (!resize_supported) + total_size = (cq_pages << ZXDH_HW_PAGE_SHIFT) + + ZXDH_DB_SHADOW_AREA_SIZE; + + iwucq->buf_size = total_size; +- info.cq_base = zxdh_alloc_hw_buf(total_size); ++ info.cq_base = (struct zxdh_cqe *)zxdh_alloc_hw_buf(total_size); + if (!info.cq_base) + goto err_cq_base; + +@@ -436,15 +598,14 @@ + + iwucq->vmr.ibv_mr.pd = &iwvctx->iwupd->ibv_pd; + +- if (dev_attrs->feature_flags & ZXDH_FEATURE_CQ_RESIZE) { +- info.shadow_area = zxdh_alloc_hw_buf(ZXDH_DB_SHADOW_AREA_SIZE); ++ if (resize_supported) { ++ info.shadow_area = (__le64 *)zxdh_alloc_hw_buf(ZXDH_DB_SHADOW_AREA_SIZE); + if (!info.shadow_area) + goto err_dereg_mr; + + memset(info.shadow_area, 0, ZXDH_DB_SHADOW_AREA_SIZE); + reg_mr_shadow_cmd.reg_type = ZXDH_MEMREG_TYPE_CQ; + reg_mr_shadow_cmd.cq_pages = 1; +- + ret = ibv_cmd_reg_mr( + &iwvctx->iwupd->ibv_pd, info.shadow_area, + ZXDH_DB_SHADOW_AREA_SIZE, (uintptr_t)info.shadow_area, +@@ -457,7 +618,6 @@ + } + + iwucq->vmr_shadow_area.ibv_mr.pd = &iwvctx->iwupd->ibv_pd; +- + } else { + info.shadow_area = (__le64 *)((__u8 *)info.cq_base + + (cq_pages << ZXDH_HW_PAGE_SHIFT)); +@@ -466,10 +626,7 @@ + attr_ex->cqe = info.cq_size; + cmd.user_cq_buf = (__u64)((uintptr_t)info.cq_base); + cmd.user_shadow_area = (__u64)((uintptr_t)info.shadow_area); +- +- ret = ibv_cmd_create_cq_ex(context, attr_ex, &iwucq->verbs_cq, +- &cmd.ibv_cmd, sizeof(cmd), &resp.ibv_resp, +- sizeof(resp), 0); ++ ret = ibv_cmd_create_cq_ex_wrap(context, attr_ex, &iwucq->verbs_cq, &cmd.ibv_cmd, sizeof(cmd), &resp.ibv_resp, sizeof(resp)); + if (ret) { + errno = ret; + goto err_dereg_shadow; +@@ -481,8 +638,13 @@ + /* Do not report the cqe's burned by HW */ + iwucq->verbs_cq.cq.cqe = ncqe; + ++ addr_offset = ++ (ZXDH_SYS_PAGE_SIZE == ZXDH_HW_64K_PAGE_SIZE) ? ++ (iwvctx->dev_attrs.cq_db_bar_off & (ZXDH_HW_64K_PAGE_SIZE - 1)) : ++ 0; ++ + info.cqe_alloc_db = +- (__u32 *)((__u8 *)iwvctx->cq_db + ZXDH_DB_CQ_OFFSET); ++ (__u32 *)((__u8 *)iwvctx->cq_db + ZXDH_DB_CQ_OFFSET + addr_offset); + zxdh_cq_init(&iwucq->cq, &info); + + return &iwucq->verbs_cq.cq_ex; +@@ -491,7 +653,9 @@ + ibv_cmd_dereg_mr(&iwucq->vmr); + if (iwucq->vmr_shadow_area.ibv_mr.handle) { + ibv_cmd_dereg_mr(&iwucq->vmr_shadow_area); +- zxdh_free_hw_buf(info.shadow_area, ZXDH_DB_SHADOW_AREA_SIZE); ++ if (resize_supported) ++ zxdh_free_hw_buf(info.shadow_area, ++ ZXDH_DB_SHADOW_AREA_SIZE); + } + err_dereg_mr: + zxdh_free_hw_buf(info.cq_base, total_size); +@@ -587,12 +751,17 @@ + goto err; + + get_64bit_val(iwucq->cq.shadow_area, 0, &cq_shadow_temp); +- +- zxdh_process_resize_list(iwucq, NULL); ++ if (!iwucq->is_ext_buf_mode) ++ zxdh_process_resize_list(iwucq, NULL); + ret = ibv_cmd_destroy_cq(cq); + if (ret) + goto err; + ++ if (iwucq->is_ext_buf_mode) { ++ free(iwucq); ++ return 0; ++ } ++ + ibv_cmd_dereg_mr(&iwucq->vmr); + zxdh_free_hw_buf(iwucq->cq.cq_base, iwucq->buf_size); + +@@ -615,31 +784,50 @@ + return ibv_cmd_modify_cq(cq, attr, &cmd, sizeof(cmd)); + } + +-static enum ibv_wc_status +-zxdh_flush_err_to_ib_wc_status(enum zxdh_flush_opcode opcode) ++static enum ibv_wc_status zxdh_err_to_ib_wc_status(__u32 opcode) + { + switch (opcode) { +- case FLUSH_PROT_ERR: ++ case ZXDH_RX_WQE_LEN_ERR: ++ return IBV_WC_LOC_LEN_ERR; ++ case ZXDH_TX_ACK_SYS_TOP_VADDR_LEN_CHECK_ERR: ++ case ZXDH_TX_ACK_SYS_TOP_LKEY_CHECK_ERR: ++ case ZXDH_TX_ACK_SYS_TOP_ACCESS_RIGHT_CHECK_ERR: ++ case ZXDH_RX_MR_MW_STATE_FREE_ERR: ++ case ZXDH_RX_MR_MW_STATE_INVALID_ERR: ++ case ZXDH_RX_MR_MW_PD_CHECK_ERR: ++ case ZXDH_RX_MR_MW_KEY_CHECK_ERR: ++ case ZXDH_RX_MR_MW_STAG_INDEX_CHECK_ERR: ++ case ZXDH_RX_MR_MW_BOUNDARY_CHECK_ERR: ++ case ZXDH_RX_MR_MW_0STAG_INDEX_CHECK_ERR: ++ case ZXDH_RX_MW_STATE_INVALID_ERR: ++ case ZXDH_RX_MW_PD_CHECK_ERR: ++ case ZXDH_RX_MW_STAG_INDEX_CHECK_ERR: ++ case ZXDH_RX_MW_SHARE_MR_CHECK_ERR: ++ case ZXDH_RX_MR_PD_CHECK_ERR: ++ case ZXDH_RX_MR_SHARE_MR_CHECK_ERR: ++ case ZXDH_RX_MR_MW_ACCESS_CHECK_ERR: + return IBV_WC_LOC_PROT_ERR; +- case FLUSH_REM_ACCESS_ERR: ++ case ZXDH_TX_PARSE_TOP_WQE_FLUSH: ++ return IBV_WC_WR_FLUSH_ERR; ++ case ZXDH_TX_ACK_SYS_TOP_NAK_INVALID_REQ: ++ return IBV_WC_REM_INV_REQ_ERR; ++ case ZXDH_TX_ACK_SYS_TOP_NAK_REMOTE_ACCESS_ERR: ++ case ZXDH_RX_MW_RKEY_CHECK_ERR: ++ case ZXDH_RX_MR_RKEY_CHECK_ERR: + return IBV_WC_REM_ACCESS_ERR; +- case FLUSH_LOC_QP_OP_ERR: +- return IBV_WC_LOC_QP_OP_ERR; +- case FLUSH_REM_OP_ERR: ++ case ZXDH_TX_ACK_SYS_TOP_NAK_REMOTE_OPERATIONAL_ERR: + return IBV_WC_REM_OP_ERR; +- case FLUSH_LOC_LEN_ERR: +- return IBV_WC_LOC_LEN_ERR; +- case FLUSH_GENERAL_ERR: +- return IBV_WC_WR_FLUSH_ERR; +- case FLUSH_RETRY_EXC_ERR: ++ case ZXDH_TX_ACK_SYS_TOP_NAK_RETRY_LIMIT: ++ case ZXDH_TX_ACK_SYS_TOP_READ_RETRY_LIMIT: ++ case ZXDH_TX_ACK_SYS_TOP_TIMEOUT_RETRY_LIMIT: + return IBV_WC_RETRY_EXC_ERR; +- case FLUSH_MW_BIND_ERR: +- return IBV_WC_MW_BIND_ERR; +- case FLUSH_REM_INV_REQ_ERR: +- return IBV_WC_REM_INV_REQ_ERR; +- case FLUSH_FATAL_ERR: +- default: ++ case ZXDH_TX_ACK_SYS_TOP_RNR_RETRY_LIMIT: ++ return IBV_WC_RNR_RETRY_EXC_ERR; ++ case ZXDH_TX_PARSE_TOP_AXI_ERR: ++ case ZXDH_RX_AXI_RESP_ERR: + return IBV_WC_FATAL_ERR; ++ default: ++ return IBV_WC_GENERAL_ERR; + } + } + +@@ -655,11 +843,8 @@ + + ibvcq_ex->wr_id = cur_cqe->wr_id; + if (cur_cqe->error) +- ibvcq_ex->status = +- (cur_cqe->comp_status == ZXDH_COMPL_STATUS_FLUSHED) ? +- zxdh_flush_err_to_ib_wc_status( +- cur_cqe->minor_err) : +- IBV_WC_GENERAL_ERR; ++ ibvcq_ex->status = zxdh_err_to_ib_wc_status( ++ cur_cqe->major_err << 16 | cur_cqe->minor_err); + else + ibvcq_ex->status = IBV_WC_SUCCESS; + } +@@ -682,11 +867,8 @@ + ib_qp = qp->back_qp; + + if (cur_cqe->error) { +- entry->status = +- (cur_cqe->comp_status == ZXDH_COMPL_STATUS_FLUSHED) ? +- zxdh_flush_err_to_ib_wc_status( +- cur_cqe->minor_err) : +- IBV_WC_GENERAL_ERR; ++ entry->status = zxdh_err_to_ib_wc_status( ++ cur_cqe->major_err << 16 | cur_cqe->minor_err); + entry->vendor_err = + cur_cqe->major_err << 16 | cur_cqe->minor_err; + } else { +@@ -719,6 +901,10 @@ + case ZXDH_OP_TYPE_LOCAL_INV: + entry->opcode = IBV_WC_LOCAL_INV; + break; ++ case ZXDH_OP_TYPE_REC_IMM: ++ entry->opcode = IBV_WC_RECV; ++ entry->wc_flags |= IBV_WC_WITH_IMM; ++ break; + case ZXDH_OP_TYPE_REC: + entry->opcode = IBV_WC_RECV; + if (ib_qp->qp_type != IBV_QPT_UD && cur_cqe->stag_invalid_set) { +@@ -726,12 +912,9 @@ + entry->wc_flags |= IBV_WC_WITH_INV; + } + break; +- case ZXDH_OP_TYPE_REC_IMM: ++ case ZXDH_OP_TYPE_REC_WRITE_IMM: + entry->opcode = IBV_WC_RECV_RDMA_WITH_IMM; +- if (ib_qp->qp_type != IBV_QPT_UD && cur_cqe->stag_invalid_set) { +- entry->invalidated_rkey = cur_cqe->inv_stag; +- entry->wc_flags |= IBV_WC_WITH_INV; +- } ++ entry->wc_flags |= IBV_WC_WITH_IMM; + break; + default: + entry->status = IBV_WC_GENERAL_ERR; +@@ -811,6 +994,7 @@ + cq_new_cqe = true; + continue; + } ++ printf("__zrdma_upoll_cq resize goto error failed\n"); + goto error; + } + +@@ -838,6 +1022,7 @@ + cq_new_cqe = true; + continue; + } ++ printf("__zrdma_upoll_cq goto error failed\n"); + goto error; + } + if (cq_new_cqe) +@@ -877,10 +1062,14 @@ + while (npolled < num_entries) { + ret = zxdh_poll_one(&iwucq->cq, cur_cqe, + entry ? entry + npolled : NULL); ++ if (ret == ZXDH_ERR_SRQ_EMPTY || ret == ZXDH_ERR_Q_DESTROYED) ++ continue; ++ + if (unlikely(ret != ZXDH_SUCCESS)) + break; + ++npolled; + } ++ + return npolled; + } + +@@ -1020,7 +1209,7 @@ + container_of(ibvcq_ex, struct zxdh_ucq, verbs_cq.cq_ex); + + /* RTT is in usec */ +- return iwucq->cur_cqe.tcp_seq_num_rtt * 1000; ++ return (uint64_t)iwucq->cur_cqe.tcp_seq_num_rtt * 1000; + } + + static enum ibv_wc_opcode zxdh_wc_read_opcode(struct ibv_cq_ex *ibvcq_ex) +@@ -1042,9 +1231,10 @@ + return IBV_WC_SEND; + case ZXDH_OP_TYPE_BIND_MW: + return IBV_WC_BIND_MW; ++ case ZXDH_OP_TYPE_REC_IMM: + case ZXDH_OP_TYPE_REC: + return IBV_WC_RECV; +- case ZXDH_OP_TYPE_REC_IMM: ++ case ZXDH_OP_TYPE_REC_WRITE_IMM: + return IBV_WC_RECV_RDMA_WITH_IMM; + case ZXDH_OP_TYPE_LOCAL_INV: + return IBV_WC_LOCAL_INV; +@@ -1089,9 +1279,6 @@ + case ZXDH_OP_TYPE_REC: + wc_flags |= IBV_WC_WITH_INV; + break; +- case ZXDH_OP_TYPE_REC_IMM: +- wc_flags |= IBV_WC_WITH_INV; +- break; + } + } + } +@@ -1258,27 +1445,30 @@ + pthread_spin_unlock(&iwucq->lock); + } + +-void *zxdh_mmap(int fd, off_t offset) ++void *zxdh_mmap(int fd, __u32 len, off_t offset) + { +- void *map; +- +- map = mmap(NULL, ZXDH_HW_PAGE_SIZE, PROT_WRITE | PROT_READ, MAP_SHARED, +- fd, offset); +- if (map == MAP_FAILED) +- return map; ++ __u64 mmap_len = ++ (ZXDH_SYS_PAGE_SIZE == ZXDH_HW_64K_PAGE_SIZE) ? ++ ZXDH_HW_64K_PAGE_SIZE : ++ len; + +- if (ibv_dontfork_range(map, ZXDH_HW_PAGE_SIZE)) { +- munmap(map, ZXDH_HW_PAGE_SIZE); +- return MAP_FAILED; +- } +- +- return map; ++ return mmap(NULL, mmap_len, PROT_WRITE | PROT_READ, MAP_SHARED, ++ fd, offset); + } + +-void zxdh_munmap(void *map) ++void zxdh_munmap(void *map, __u32 len) + { +- ibv_dofork_range(map, ZXDH_HW_PAGE_SIZE); +- munmap(map, ZXDH_HW_PAGE_SIZE); ++ __u64 mmap_len = ++ (ZXDH_SYS_PAGE_SIZE == ZXDH_HW_64K_PAGE_SIZE) ? ++ ZXDH_HW_64K_PAGE_SIZE : ++ len; ++ ++ if (map == NULL) { ++ return; ++ } ++ ++ ibv_dofork_range(map, mmap_len); ++ munmap(map, mmap_len); + } + + /** +@@ -1292,8 +1482,9 @@ + ret = ibv_cmd_destroy_qp(&iwuqp->vqp.qp); + if (ret) + return ret; +- ++ if (iwuqp->ext_buf_mode == 0){ + ibv_cmd_dereg_mr(&iwuqp->vmr); ++ } + + return 0; + } +@@ -1324,13 +1515,13 @@ + rqsize = 0; + sqsize = roundup(sqdepth * ZXDH_QP_SQE_MIN_SIZE, ZXDH_HW_PAGE_SIZE); + if (iwuqp->is_srq == false) { +- rqsize = roundup(rqdepth * ZXDH_QP_RQE_MIN_SIZE, ++ rqsize = roundup((rqdepth << info->rqshift) * ZXDH_QP_RQE_MIN_SIZE, + ZXDH_HW_PAGE_SIZE); + totalqpsize = rqsize + sqsize + ZXDH_DB_SHADOW_AREA_SIZE; + } else { + totalqpsize = sqsize + ZXDH_DB_SHADOW_AREA_SIZE; + } +- info->sq = zxdh_alloc_hw_buf(totalqpsize); ++ info->sq = (struct zxdh_qp_sq_quanta *)zxdh_alloc_hw_buf(totalqpsize); + iwuqp->buf_size = totalqpsize; + + if (!info->sq) +@@ -1349,7 +1540,6 @@ + } + reg_mr_cmd.reg_type = ZXDH_MEMREG_TYPE_QP; + reg_mr_cmd.sq_pages = sqsize >> ZXDH_HW_PAGE_SHIFT; +- + ret = ibv_cmd_reg_mr(pd, info->sq, totalqpsize, (uintptr_t)info->sq, + IBV_ACCESS_LOCAL_WRITE, &iwuqp->vmr, + ®_mr_cmd.ibv_cmd, sizeof(reg_mr_cmd), +@@ -1359,18 +1549,33 @@ + + cmd.user_wqe_bufs = (__u64)((uintptr_t)info->sq); + cmd.user_compl_ctx = (__u64)(uintptr_t)&iwuqp->qp; ++ ++ if (info->is_qp_ext_mem_support) { ++ cmd.support_ext_qp = 1; ++ } + ret = ibv_cmd_create_qp(pd, &iwuqp->vqp.qp, attr, &cmd.ibv_cmd, + sizeof(cmd), &resp.ibv_resp, + sizeof(struct zxdh_ucreate_qp_resp)); + if (ret) + goto err_qp; ++#if defined(__x86_64__) || defined (__i386__) ++ if (resp.wqe_rate_limit_flag) { ++ rdma_wqe_rate_limit_qp_init(info, pd->context->device->ibdev_path, iwuqp, resp.wqe_rate_limit_valid); ++ } ++#endif + + info->sq_size = resp.actual_sq_size; + info->rq_size = resp.actual_rq_size; ++ + info->qp_caps = resp.qp_caps; + info->qp_id = resp.qp_id; + iwuqp->zxdh_drv_opt = resp.zxdh_drv_opt; + iwuqp->vqp.qp.qp_num = resp.qp_id; ++ iwuqp->qp.is_ext_qp = resp.is_ext_qp; ++ if (iwuqp->qp.is_ext_qp) { ++ iwuqp->qp.ext_sq_base_va = (struct zxdh_qp_sq_quanta *)((__u8 *)info->ext_qp_buff_base + resp.sq_offset); ++ iwuqp->qp.ext_rq_base_va = (struct zxdh_qp_rq_quanta *)((__u8 *)info->ext_qp_buff_base + resp.rq_offset); ++ } + + iwuqp->send_cq = + container_of(attr->send_cq, struct zxdh_ucq, verbs_cq.cq); +@@ -1386,7 +1591,7 @@ + zxdh_free_hw_buf(info->sq, iwuqp->buf_size); + return ret; + } +- ++#if ZXDH_HAVE_QP_EX + static void zxdh_wr_local_inv(struct ibv_qp_ex *ibqp, uint32_t invalidate_rkey) + { + struct zxdh_uqp *qp = container_of(ibqp, struct zxdh_uqp, vqp.qp_ex); +@@ -1429,7 +1634,7 @@ + + zxdh_upost_send(&qp->vqp.qp, &wr, &bad_wr); + } +- ++#endif + static struct ibv_qp *create_qp(struct ibv_context *ibv_ctx, + struct ibv_qp_init_attr_ex *attr_ex) + { +@@ -1443,6 +1648,8 @@ + __u32 sqdepth, rqdepth; + __u8 sqshift, rqshift; + int status; ++ __u64 addr_offset = 0; ++ int ret; + + attr = calloc(1, sizeof(*attr)); + if (!attr) +@@ -1455,7 +1662,6 @@ + free(attr); + return NULL; + } +- + iwvctx = container_of(ibv_ctx, struct zxdh_uvcontext, ibv_ctx.context); + dev_attrs = &iwvctx->dev_attrs; + +@@ -1466,9 +1672,15 @@ + return NULL; + } + ++ if (iwvctx->is_qp_ext_mem_support && attr->cap.max_recv_sge > dev_attrs->max_hw_rq_sges) { ++ zxdh_dbg(ZXDH_DBG_SRQ, "zxdh_check_rq_init_attr max_recv_sge failed, when use_blue_flame=true, max_recv_sge is 7.\n"); ++ errno = EINVAL; ++ free(attr); ++ return NULL; ++ } ++ + if (attr->cap.max_inline_data > dev_attrs->max_hw_inline) { +- zxdh_dbg(&iwvctx->ibv_ctx, ZXDH_DBG_QP, +- "max_inline_data over max_hw_inline\n"); ++ zxdh_dbg(ZXDH_DBG_QP, "max_inline_data over max_hw_inline\n"); + attr->cap.max_inline_data = dev_attrs->max_hw_inline; + } + +@@ -1498,7 +1710,8 @@ + } + + memset(iwuqp, 0, sizeof(*iwuqp)); +- ++ iwuqp->wqe_rate_limit_info = NULL; ++#if ZXDH_HAVE_QP_EX + if (attr_ex->comp_mask & IBV_QP_INIT_ATTR_SEND_OPS_FLAGS) { + if (attr_ex->send_ops_flags & ~IBV_QP_EX_WITH_BIND_MW) { + errno = EOPNOTSUPP; +@@ -1517,12 +1730,16 @@ + if (attr_ex->send_ops_flags & IBV_QP_EX_WITH_LOCAL_INV) + iwuqp->vqp.qp_ex.wr_local_inv = zxdh_wr_local_inv; + } +- ++#endif + if (pthread_spin_init(&iwuqp->lock, PTHREAD_PROCESS_PRIVATE)) + goto err_free_qp; +- ++ if (pthread_spin_init(&iwuqp->quanta_lock, PTHREAD_PROCESS_PRIVATE)) ++ goto err_destroy_quanta_lock; ++ // sqdepth is the number of quanta, not the wqe num ++ // rqdepth is the wqe num, not the number of quanta + info.sq_size = sqdepth >> sqshift; +- info.rq_size = rqdepth >> rqshift; ++ info.rq_size = rqdepth; ++ info.rqshift = rqshift; + attr->cap.max_send_wr = info.sq_size; + attr->cap.max_recv_wr = info.rq_size; + +@@ -1544,8 +1761,17 @@ + goto err_destroy_lock; + } + ++ addr_offset = ++ (ZXDH_SYS_PAGE_SIZE == ZXDH_HW_64K_PAGE_SIZE) ? ++ (iwvctx->dev_attrs.sq_db_bar_off & (ZXDH_HW_64K_PAGE_SIZE - 1)) : ++ 0; ++ if (iwvctx->is_qp_ext_mem_support) { ++ info.is_qp_ext_mem_support = iwvctx->is_qp_ext_mem_support; ++ info.ext_sq_db = (__u32 *)((__u8 *)iwvctx->ext_sq_db + ZXDH_DB_SQ_OFFSET); ++ info.ext_qp_buff_base = iwvctx->ext_qp_buff_base; ++ } + info.wqe_alloc_db = +- (__u32 *)((__u8 *)iwvctx->sq_db + ZXDH_DB_SQ_OFFSET); ++ (__u32 *)((__u8 *)iwvctx->sq_db + ZXDH_DB_SQ_OFFSET + addr_offset); + info.abi_ver = iwvctx->abi_ver; + info.legacy_mode = iwvctx->legacy_mode; + info.sq_wrtrk_array = calloc(sqdepth, sizeof(*info.sq_wrtrk_array)); +@@ -1574,6 +1800,7 @@ + + iwuqp->qp.back_qp = iwuqp; + iwuqp->qp.lock = &iwuqp->lock; ++ iwuqp->qp.quanta_lock = &iwuqp->quanta_lock; + info.max_sq_frag_cnt = attr->cap.max_send_sge; + info.max_rq_frag_cnt = attr->cap.max_recv_sge; + info.max_inline_data = attr->cap.max_inline_data; +@@ -1590,8 +1817,8 @@ + goto err_free_sg_list; + } + iwuqp->qp.mtu = mtu_enum_to_int(IBV_MTU_1024); +- attr->cap.max_send_wr = (sqdepth - ZXDH_SQ_RSVD) >> sqshift; +- attr->cap.max_recv_wr = (rqdepth - ZXDH_RQ_RSVD) >> rqshift; ++ attr->cap.max_send_wr = sqdepth >> sqshift; ++ attr->cap.max_recv_wr = rqdepth; + memcpy(attr_ex, attr, sizeof(*attr)); + free(attr); + return &iwuqp->vqp.qp; +@@ -1609,6 +1836,10 @@ + err_free_rsges: + free(iwuqp->recv_sges); + err_destroy_lock: ++ ret = pthread_spin_destroy(&iwuqp->quanta_lock); ++ if (ret) ++ errno = EINVAL; ++err_destroy_quanta_lock: + pthread_spin_destroy(&iwuqp->lock); + err_free_qp: + free(iwuqp); +@@ -1666,6 +1897,40 @@ + } + + /** ++ * zxdh_clean_cqes - clean cq entries for qp ++ * @qp: qp for which completions are cleaned ++ * @iwcq: cq to be cleaned ++ */ ++static void zxdh_clean_cqes(struct zxdh_qp *qp, struct zxdh_ucq *iwucq) ++{ ++ struct zxdh_cq *ukcq = &iwucq->cq; ++ int ret; ++ ++ ret = pthread_spin_lock(&iwucq->lock); ++ if (ret) ++ return; ++ ++ zxdh_clean_cq(qp, ukcq); ++ pthread_spin_unlock(&iwucq->lock); ++} ++ ++static void zxdh_init_qp_indices(struct zxdh_qp *qp) ++{ ++ __u32 sq_ring_size; ++ sq_ring_size = ZXDH_RING_SIZE(qp->sq_ring); ++ ZXDH_RING_INIT(qp->sq_ring, sq_ring_size); ++ ZXDH_RING_INIT(qp->initial_ring, sq_ring_size); ++ qp->swqe_polarity = 0; ++ qp->swqe_polarity_deferred = 1; ++ qp->rwqe_polarity = 0; ++ qp->rwqe_signature = 0; ++ ZXDH_RING_INIT(qp->rq_ring, qp->rq_size); ++ /* 新增：初始化last_sqe_idx和last_rqe_idx */ ++ qp->last_sqe_idx = 0; ++ qp->last_rqe_idx = 0; ++} ++ ++/** + * zxdh_umodify_qp - send qp modify to driver + * @qp: qp to modify + * @attr: attribute to modify +@@ -1681,6 +1946,14 @@ + __u16 mtu = 0; + + iwuqp = container_of(qp, struct zxdh_uqp, vqp.qp); ++ if (attr_mask & IBV_QP_CAP) ++ return -EINVAL; ++ if (attr_mask & IBV_QP_PATH_MTU) { ++ if (attr->path_mtu < IBV_MTU_256 || attr->path_mtu > IBV_MTU_4096) { ++ return -EINVAL; ++ } ++ } ++ + if (attr_mask & IBV_QP_STATE || attr_mask & IBV_QP_RATE_LIMIT) { + ret = ibv_cmd_modify_qp_ex(qp, attr, attr_mask, &cmd_ex.ibv_cmd, + sizeof(cmd_ex), &resp.ibv_resp, +@@ -1688,6 +1961,18 @@ + } else { + ret = ibv_cmd_modify_qp(qp, attr, attr_mask, &cmd, sizeof(cmd)); + } ++ ++ if (!ret && (attr_mask & IBV_QP_STATE) && ++ attr->qp_state == IBV_QPS_RESET) { ++ if (iwuqp->send_cq) ++ zxdh_clean_cqes(&iwuqp->qp, iwuqp->send_cq); ++ ++ if (iwuqp->recv_cq && iwuqp->recv_cq != iwuqp->send_cq) ++ zxdh_clean_cqes(&iwuqp->qp, iwuqp->recv_cq); ++ memset(iwuqp->qp.sq_base, 0, iwuqp->buf_size); ++ zxdh_init_qp_indices(&iwuqp->qp); ++ } ++ + if (!ret && (attr_mask & IBV_QP_PATH_MTU) && + qp->qp_type == IBV_QPT_RC) { + mtu = mtu_enum_to_int(attr->path_mtu); +@@ -1720,24 +2005,6 @@ + } + + /** +- * zxdh_clean_cqes - clean cq entries for qp +- * @qp: qp for which completions are cleaned +- * @iwcq: cq to be cleaned +- */ +-static void zxdh_clean_cqes(struct zxdh_qp *qp, struct zxdh_ucq *iwucq) +-{ +- struct zxdh_cq *cq = &iwucq->cq; +- int ret; +- +- ret = pthread_spin_lock(&iwucq->lock); +- if (ret) +- return; +- +- zxdh_clean_cq(qp, cq); +- pthread_spin_unlock(&iwucq->lock); +-} +- +-/** + * zxdh_udestroy_qp - destroy qp + * @qp: qp to destroy + */ +@@ -1747,16 +2014,25 @@ + int ret; + + iwuqp = container_of(qp, struct zxdh_uqp, vqp.qp); ++ ret = pthread_spin_destroy(&iwuqp->quanta_lock); ++ if (ret) ++ goto err; ++ iwuqp->qp.quanta_lock = NULL; + ret = pthread_spin_destroy(&iwuqp->lock); + if (ret) + goto err; ++ iwuqp->qp.lock = NULL; + + iwuqp->qp.destroy_pending = true; + + ret = zxdh_destroy_vmapped_qp(iwuqp); + if (ret) + goto err; +- ++#if defined(__x86_64__) || defined (__i386__) ++ if (iwuqp->wqe_rate_limit_info) { ++ rdma_wqe_rate_limit_qp_exit(iwuqp); ++ } ++#endif + /* Clean any pending completions from the cq(s) */ + if (iwuqp->send_cq) + zxdh_clean_cqes(&iwuqp->qp, iwuqp->send_cq); +@@ -1771,8 +2047,12 @@ + if (iwuqp->qp.split_sg_list) + free(iwuqp->qp.split_sg_list); + +- zxdh_free_hw_buf(iwuqp->qp.sq_base, iwuqp->buf_size); +- free(iwuqp->recv_sges); ++ if (iwuqp->ext_buf_mode == 0) ++ zxdh_free_hw_buf(iwuqp->qp.sq_base, iwuqp->buf_size); ++ ++ if (iwuqp->recv_sges) ++ free(iwuqp->recv_sges); ++ + free(iwuqp); + return 0; + +@@ -1830,20 +2110,15 @@ + struct zxdh_uqp *iwuqp; + bool reflush = false; + int err = 0; ++ errno = 0; + struct verbs_mr *vmr = NULL; + struct zxdh_umr *umr = NULL; + __u64 mr_va = 0, mw_va = 0, value_dffer = 0, mw_pa_pble_index = 0; + __u16 mr_offset = 0; +- iwvctx = container_of(ib_qp->context, struct zxdh_uvcontext, +- ibv_ctx.context); +- if (ib_qp->state != IBV_QPS_RTS) { +- *bad_wr = ib_wr; +- verbs_err(&iwvctx->ibv_ctx, "zrdma: post send at state:%d\n", +- ib_qp->state); +- return -EINVAL; +- } + + iwuqp = container_of(ib_qp, struct zxdh_uqp, vqp.qp); ++ iwvctx = container_of(ib_qp->context, struct zxdh_uvcontext, ++ ibv_ctx.context); + dev_attrs = &iwvctx->dev_attrs; + + err = pthread_spin_lock(&iwuqp->lock); +@@ -1869,7 +2144,9 @@ + info.imm_data_valid = true; + info.imm_data = ntohl(ib_wr->imm_data); + } else { +- err = EINVAL; ++ zxdh_status_to_errno_log(ZXDH_NOT_SUPPORTED, ++ "qp not supported send with imm, qp_id=%d", ++ iwuqp->qp.qp_id); + break; + } + SWITCH_FALLTHROUGH; +@@ -1901,11 +2178,8 @@ + iwuqp->inline_data, ib_wr, + &info.op.inline_rdma_send.len); + if (ret) { +- verbs_err( +- &iwvctx->ibv_ctx, +- "zrdma: get inline data fail\n"); +- pthread_spin_unlock(&iwuqp->lock); +- return -EINVAL; ++ zxdh_status_to_errno_log(ret,"qp get inline data failed, qp_id=%d", iwuqp->qp.qp_id); ++ break; + } + info.op.inline_rdma_send.data = + iwuqp->inline_data; +@@ -1950,16 +2224,16 @@ + } + } + if (ret) +- err = (ret == ZXDH_ERR_QP_TOOMANY_WRS_POSTED) ? +- ENOMEM : +- EINVAL; ++ zxdh_status_to_errno_log(ret,"qp send data failed, qp_id=%d", iwuqp->qp.qp_id); + break; + case IBV_WR_RDMA_WRITE_WITH_IMM: + if (iwuqp->qp.qp_caps & ZXDH_WRITE_WITH_IMM) { + info.imm_data_valid = true; + info.imm_data = ntohl(ib_wr->imm_data); + } else { +- err = -EINVAL; ++ zxdh_status_to_errno_log(ZXDH_NOT_SUPPORTED, ++ "qp not supported write with imm, qp_id=%d", ++ iwuqp->qp.qp_id); + break; + } + SWITCH_FALLTHROUGH; +@@ -1978,11 +2252,8 @@ + iwuqp->inline_data, ib_wr, + &info.op.inline_rdma_write.len); + if (ret) { +- verbs_err( +- &iwvctx->ibv_ctx, +- "zrdma: get inline data fail\n"); +- pthread_spin_unlock(&iwuqp->lock); +- return -EINVAL; ++ zxdh_status_to_errno_log(ret,"qp get inline data failed, qp_id=%d", iwuqp->qp.qp_id); ++ break; + } + info.op.inline_rdma_write.data = + iwuqp->inline_data; +@@ -2003,13 +2274,11 @@ + ret = zxdh_rdma_write(&iwuqp->qp, &info, false); + } + if (ret) +- err = (ret == ZXDH_ERR_QP_TOOMANY_WRS_POSTED) ? +- ENOMEM : +- EINVAL; ++ zxdh_status_to_errno_log(ret,"qp write data failed, qp_id=%d", iwuqp->qp.qp_id); + break; + case IBV_WR_RDMA_READ: + if (ib_wr->num_sge > dev_attrs->max_hw_read_sges) { +- err = EINVAL; ++ zxdh_status_to_errno_log(ZXDH_ERR_INVALID_FRAG_COUNT,"qp invalid sge number, qp_id=%d", iwuqp->qp.qp_id); + break; + } + info.op_type = ZXDH_OP_TYPE_READ; +@@ -2018,12 +2287,15 @@ + info.op.rdma_read.rem_addr.stag = ib_wr->wr.rdma.rkey; + + info.op.rdma_read.lo_sg_list = (void *)ib_wr->sg_list; +- info.op.rdma_read.num_lo_sges = ib_wr->num_sge; ++ info.op.rdma_read.num_lo_sges = ib_wr->num_sge; ++#if defined(__x86_64__) || defined (__i386__) ++ if (iwuqp->wqe_rate_limit_info && iwuqp->wqe_rate_limit_info->wqe_rate_limit) { ++ rdma_qp_delay_wqe(&info, iwuqp); ++ } ++#endif + ret = zxdh_rdma_read(&iwuqp->qp, &info, false, false); + if (ret) +- err = (ret == ZXDH_ERR_QP_TOOMANY_WRS_POSTED) ? +- ENOMEM : +- EINVAL; ++ zxdh_status_to_errno_log(ret,"qp read date failed, qp_id=%d", iwuqp->qp.qp_id); + break; + case IBV_WR_BIND_MW: + vmr = verbs_get_mr(ib_wr->bind_mw.bind_info.mr); +@@ -2035,7 +2307,7 @@ + mw_pa_pble_index = 0; + + if (ib_qp->qp_type != IBV_QPT_RC) { +- err = EINVAL; ++ zxdh_status_to_errno_log(ZXDH_NOT_SUPPORTED,"qp type not supported bind mw, qp_id=%d", iwuqp->qp.qp_id); + break; + } + info.op_type = ZXDH_OP_TYPE_BIND_MW; +@@ -2061,7 +2333,7 @@ + info.op.bind_window.addressing_type = + ZXDH_ADDR_TYPE_ZERO_BASED; + if (ib_wr->bind_mw.mw->type == IBV_MW_TYPE_1) { +- err = EINVAL; ++ zxdh_status_to_errno_log(ZXDH_ERR_BIND_MW_TYPE,"bind mw type not supported bind mw, qp_id=%d", iwuqp->qp.qp_id); + break; + } + +@@ -2127,7 +2399,59 @@ + info.op.bind_window + .root_leaf_offset = 0; + } +- ++ } else if (umr->host_page_size == ZXDH_PAGE_SIZE_64K) { ++ mr_offset = mr_va & 0xFFFF; ++ value_dffer = mw_va - mr_va; ++ if (umr->leaf_pbl_size == 3) { ++ mw_pa_pble_index = ++ (mr_offset + ++ value_dffer) / ++ ((4096 * 16) * 512); ++ info.op.bind_window ++ .mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ mw_pa_pble_index; ++ mw_pa_pble_index = ++ ((mr_offset + ++ value_dffer) / ++ (4096 * 16)) % 512; ++ info.op.bind_window ++ .root_leaf_offset = ++ (__u16)mw_pa_pble_index; ++ info.op.bind_window.va = ++ (void *)(uintptr_t)(mw_va & ++ 0xFFFF); ++ info.op.bind_window ++ .leaf_pbl_size = 3; ++ } else if (umr->leaf_pbl_size == 1) { ++ mw_pa_pble_index = ++ (mr_offset + ++ value_dffer) / ++ (4096 * 16); ++ info.op.bind_window ++ .mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ mw_pa_pble_index; ++ info.op.bind_window ++ .leaf_pbl_size = 1; ++ info.op.bind_window.va = ++ (void *)(uintptr_t)(mw_va & ++ 0xFFFF); ++ } else { ++ mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ mr_offset + value_dffer; ++ info.op.bind_window.va = ++ (void *)(uintptr_t)(mw_va & ++ 0xFFFF); ++ info.op.bind_window ++ .mw_pa_pble_index = ++ mw_pa_pble_index; ++ info.op.bind_window ++ .leaf_pbl_size = 0; ++ info.op.bind_window ++ .root_leaf_offset = 0; ++ } + } else if (umr->host_page_size == + ZXDH_PAGE_SIZE_2M) { + mr_offset = mr_va & 0x1FFFFF; +@@ -2277,6 +2601,50 @@ + info.op.bind_window + .root_leaf_offset = 0; + } ++ } else if (umr->host_page_size == ZXDH_PAGE_SIZE_64K) { ++ mr_offset = mr_va & 0xFFFF; ++ value_dffer = mw_va - mr_va; ++ if (umr->leaf_pbl_size == 3) { ++ mw_pa_pble_index = ++ (mr_offset + ++ value_dffer) / ++ (4096 * 16); ++ info.op.bind_window ++ .mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ mw_pa_pble_index; ++ mw_pa_pble_index = ++ ((mr_offset + ++ value_dffer) / ++ (4096 * 16)) % ++ 512; ++ info.op.bind_window ++ .root_leaf_offset = ++ (__u16)mw_pa_pble_index; ++ info.op.bind_window ++ .leaf_pbl_size = 3; ++ } else if (umr->leaf_pbl_size == 1) { ++ info.op.bind_window ++ .mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ ((mr_offset + ++ value_dffer) / ++ (4096 * 16)); ++ info.op.bind_window ++ .leaf_pbl_size = 1; ++ info.op.bind_window ++ .root_leaf_offset = 0; ++ } else { ++ info.op.bind_window ++ .leaf_pbl_size = 0; ++ info.op.bind_window ++ .mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ (mr_va & 0xFFFF) + ++ (mw_va - mr_va); ++ info.op.bind_window ++ .root_leaf_offset = 0; ++ } + } else if (umr->host_page_size == + ZXDH_PAGE_SIZE_2M) { + mr_offset = mr_va & 0x1FFFFF; +@@ -2366,9 +2734,8 @@ + + ret = zxdh_mw_bind(&iwuqp->qp, &info, false); + if (ret) +- err = (ret == ZXDH_ERR_QP_TOOMANY_WRS_POSTED) ? +- ENOMEM : +- EINVAL; ++ zxdh_status_to_errno_log(ret,"bind mw failed, qp_id=%d", iwuqp->qp.qp_id); ++ + break; + case IBV_WR_LOCAL_INV: + info.op_type = ZXDH_OP_TYPE_LOCAL_INV; +@@ -2377,24 +2744,22 @@ + ret = zxdh_stag_local_invalidate(&iwuqp->qp, &info, + true); + if (ret) +- err = (ret == ZXDH_ERR_QP_TOOMANY_WRS_POSTED) ? +- ENOMEM : +- EINVAL; ++ zxdh_status_to_errno_log(ret,"qp local invalidate failed, qp_id=%d", iwuqp->qp.qp_id); + break; + default: + /* error */ +- err = EINVAL; ++ zxdh_status_to_errno_log(ZXDH_ERR_INVALID_WR_OP_TYPE,"invalid wr type, qp_id=%d", iwuqp->qp.qp_id); + break; + } +- if (err) ++ if (errno != 0) + break; + + ib_wr = ib_wr->next; + } + +- if (err) ++ if (errno != 0) + *bad_wr = ib_wr; +- ++ err = errno; + zxdh_qp_post_wr(&iwuqp->qp); + if (reflush) + zxdh_issue_flush(ib_qp, 1, 0); +@@ -2418,13 +2783,15 @@ + struct zxdh_uqp *iwuqp; + bool reflush = false; + int err = 0; ++ errno = 0; + + iwuqp = container_of(ib_qp, struct zxdh_uqp, vqp.qp); + sg_list = iwuqp->recv_sges; + + if (unlikely(ib_qp->state == IBV_QPS_RESET || ib_qp->srq)) { + *bad_wr = ib_wr; +- return -EINVAL; ++ zxdh_status_to_errno_log(ZXDH_ERR_BAD_QP,"post recv at reset or using srq, qp_id=%d", iwuqp->qp.qp_id); ++ return errno; + } + + err = pthread_spin_lock(&iwuqp->lock); +@@ -2438,7 +2805,7 @@ + while (ib_wr) { + if (unlikely(ib_wr->num_sge > iwuqp->qp.max_rq_frag_cnt)) { + *bad_wr = ib_wr; +- err = EINVAL; ++ zxdh_status_to_errno_log(ZXDH_ERR_INVALID_FRAG_COUNT,"qp invalid sge number, qp_id=%d", iwuqp->qp.qp_id); + goto error; + } + post_recv.num_sges = ib_wr->num_sge; +@@ -2447,8 +2814,7 @@ + post_recv.sg_list = sg_list; + ret = zxdh_post_receive(&iwuqp->qp, &post_recv); + if (unlikely(ret)) { +- err = (ret == ZXDH_ERR_QP_TOOMANY_WRS_POSTED) ? ENOMEM : +- EINVAL; ++ zxdh_status_to_errno_log(ret,"qp post recv failed, qp_id=%d", iwuqp->qp.qp_id); + *bad_wr = ib_wr; + goto error; + } +@@ -2459,6 +2825,7 @@ + ib_wr = ib_wr->next; + } + error: ++ err = errno; + zxdh_qp_set_shadow_area(&iwuqp->qp); + pthread_spin_unlock(&iwuqp->lock); + +@@ -2477,6 +2844,7 @@ + struct zxdh_ucreate_ah_resp resp; + int err; + ++ memset(&resp, 0, sizeof(resp)); + err = ibv_query_gid(ibpd->context, attr->port_num, attr->grh.sgid_index, + &sgid); + if (err) { +@@ -2568,6 +2936,7 @@ + int ret = 0; + + iwucq = container_of(cq, struct zxdh_ucq, verbs_cq.cq); ++ + iwvctx = container_of(cq->context, struct zxdh_uvcontext, + ibv_ctx.context); + dev_attrs = &iwvctx->dev_attrs; +@@ -2575,20 +2944,21 @@ + if (!(dev_attrs->feature_flags & ZXDH_FEATURE_CQ_RESIZE)) + return -EOPNOTSUPP; + +- if (cqe > ZXDH_MAX_CQ_SIZE) ++ if (cqe > dev_attrs->max_hw_cq_size) + return -EINVAL; +- +- cqe_needed = zxdh_cq_round_up(cqe + 1); ++ cqe_needed = zxdh_cq_round_up(cqe); + + if (cqe_needed < ZXDH_U_MINCQ_SIZE) + cqe_needed = ZXDH_U_MINCQ_SIZE; + +- if (cqe_needed == iwucq->cq.cq_size) +- return 0; ++ if (cqe_needed * 2 <= dev_attrs->max_hw_cq_size) ++ cqe_needed = cqe_needed * 2; ++ if (cqe == cq->cqe) ++ return 0; + + cq_size = get_cq_total_bytes(cqe_needed); + cq_pages = cq_size >> ZXDH_HW_PAGE_SHIFT; +- cq_base = zxdh_alloc_hw_buf(cq_size); ++ cq_base = (struct zxdh_cqe *)zxdh_alloc_hw_buf(cq_size); + if (!cq_base) + return -ENOMEM; + +@@ -2600,10 +2970,13 @@ + goto err_buf; + } + ++ ret = pthread_spin_lock(&iwucq->lock); ++ if (ret) ++ goto err_lock; ++ + new_mr.ibv_mr.pd = iwucq->vmr.ibv_mr.pd; + reg_mr_cmd.reg_type = ZXDH_MEMREG_TYPE_CQ; + reg_mr_cmd.cq_pages = cq_pages; +- + ret = ibv_cmd_reg_mr(new_mr.ibv_mr.pd, cq_base, cq_size, + (uintptr_t)cq_base, IBV_ACCESS_LOCAL_WRITE, + &new_mr, ®_mr_cmd.ibv_cmd, sizeof(reg_mr_cmd), +@@ -2611,10 +2984,6 @@ + if (ret) + goto err_dereg_mr; + +- ret = pthread_spin_lock(&iwucq->lock); +- if (ret) +- goto err_lock; +- + cmd.user_cq_buffer = (__u64)((uintptr_t)cq_base); + ret = ibv_cmd_resize_cq(&iwucq->verbs_cq.cq, cqe_needed, &cmd.ibv_cmd, + sizeof(cmd), &resp, sizeof(resp)); +@@ -2633,10 +3002,10 @@ + return ret; + + err_resize: +- pthread_spin_unlock(&iwucq->lock); +-err_lock: + ibv_cmd_dereg_mr(&new_mr); + err_dereg_mr: ++ pthread_spin_unlock(&iwucq->lock); ++err_lock: + free(cq_buf); + err_buf: + zxdh_free_hw_buf(cq_base, cq_size); +@@ -2651,9 +3020,8 @@ + __u64 hdr; + + srq = &iwusrq->srq; +- zxdh_dbg(verbs_get_ctx(iwusrq->ibv_srq.context), ZXDH_DBG_SRQ, +- "%s head:%d tail:%d\n", __func__, srq->srq_ring.head, +- srq->srq_ring.tail); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s head:%d tail:%d\n", __func__, ++ srq->srq_ring.head, srq->srq_ring.tail); + for (i = srq->srq_ring.head; i < srq->srq_ring.tail; i++) { + wqe = zxdh_get_srq_wqe(srq, i); + +@@ -2664,14 +3032,14 @@ + } + } + +-static size_t zxdh_get_srq_queue_size(int srqdepth) ++static size_t zxdh_get_srq_queue_size(int srqdepth, __u8 srqshift) + { +- return roundup(srqdepth * ZXDH_SRQ_WQE_MIN_SIZE, ZXDH_HW_PAGE_SIZE); ++ return roundup((srqdepth << srqshift) * ZXDH_SRQ_WQE_MIN_SIZE, ZXDH_HW_PAGE_SIZE); + } + +-static size_t zxdh_get_srq_list_size(size_t srq_size) ++static size_t zxdh_get_srq_list_size(size_t srq_size, __u8 srqshift) + { +- return roundup(srq_size * sizeof(__u16), ZXDH_HW_PAGE_SIZE); ++ return roundup((srq_size << srqshift) * sizeof(__u16), ZXDH_HW_PAGE_SIZE); + } + + static size_t zxdh_get_srq_db_size(void) +@@ -2680,16 +3048,16 @@ + } + + static size_t zxdh_get_total_srq_size(struct zxdh_usrq *iwusrq, int srqdepth, +- size_t srq_size) ++ size_t srq_size, __u8 srqshift) + { + size_t total_srq_queue_size; + size_t total_srq_list_size; + size_t total_srq_db_size; + size_t total_srq_size; + +- total_srq_queue_size = zxdh_get_srq_queue_size(srqdepth); ++ total_srq_queue_size = zxdh_get_srq_queue_size(srqdepth, srqshift); + iwusrq->buf_size = total_srq_queue_size; +- total_srq_list_size = zxdh_get_srq_list_size(srq_size); ++ total_srq_list_size = zxdh_get_srq_list_size(srq_size, srqshift); + iwusrq->list_buf_size = total_srq_list_size; + total_srq_db_size = zxdh_get_srq_db_size(); + iwusrq->db_buf_size = total_srq_db_size; +@@ -2697,7 +3065,7 @@ + total_srq_queue_size + total_srq_list_size + total_srq_db_size; + iwusrq->total_buf_size = total_srq_size; + zxdh_dbg( +- verbs_get_ctx(iwusrq->ibv_srq.context), ZXDH_DBG_SRQ, ++ ZXDH_DBG_SRQ, + "%s total_srq_queue_size:%ld total_srq_list_size:%ld total_srq_db_size:%ld srqdepth:%d\n", + __func__, total_srq_queue_size, total_srq_list_size, + total_srq_db_size, srqdepth); +@@ -2709,18 +3077,23 @@ + struct zxdh_srq_init_info *info, + size_t total_srq_size) + { +- info->srq_base = zxdh_alloc_hw_buf(total_srq_size); ++ struct zxdh_uvcontext *iwvctx; ++ iwvctx = container_of(iwusrq->ibv_srq.context, struct zxdh_uvcontext, ++ ibv_ctx.context); ++ info->srq_base = (struct zxdh_srq_wqe *)zxdh_alloc_hw_buf(total_srq_size); + if (!info->srq_base) + return -ENOMEM; + memset(info->srq_base, 0, total_srq_size); + info->srq_list_base = + (__le16 *)&info + ->srq_base[iwusrq->buf_size / ZXDH_SRQ_WQE_MIN_SIZE]; +- info->srq_db_base = +- (__le64 *)&info->srq_list_base[iwusrq->list_buf_size / +- (sizeof(__u16))]; +- *(__le64 *)info->srq_db_base = ZXDH_SRQ_DB_INIT_VALUE; +- zxdh_dbg(verbs_get_ctx(iwusrq->ibv_srq.context), ZXDH_DBG_SRQ, ++ if (iwvctx->srq_db == NULL) { ++ info->srq_db_base = ++ (__le64 *)&info->srq_list_base[iwusrq->list_buf_size / ++ (sizeof(__u16))]; ++ *(__le64 *)info->srq_db_base = ZXDH_SRQ_DB_INIT_VALUE; ++ } ++ zxdh_dbg(ZXDH_DBG_SRQ, + "%s srq_base:0x%p srq_list_base:0x%p srq_db_base:0x%p\n", + __func__, info->srq_base, info->srq_list_base, + info->srq_db_base); +@@ -2755,7 +3128,12 @@ + { + struct zxdh_ucreate_srq cmd = {}; + struct zxdh_ucreate_srq_resp resp = {}; ++ ++ struct zxdh_uvcontext *iwvctx; + int ret; ++ __u64 addr_offset = 0; ++ iwvctx = container_of(pd->context, struct zxdh_uvcontext, ++ ibv_ctx.context); + + cmd.user_wqe_bufs = (__u64)((uintptr_t)info->srq_base); + cmd.user_compl_ctx = (__u64)(uintptr_t)&iwusrq->srq; +@@ -2771,10 +3149,38 @@ + info->srq_id = resp.srq_id; + info->srq_size = resp.actual_srq_size; + info->srq_list_size = resp.actual_srq_list_size; ++ if (iwvctx->srq_db != NULL) { ++ addr_offset = ++ (ZXDH_SYS_PAGE_SIZE == ZXDH_HW_64K_PAGE_SIZE) ? ++ (iwvctx->dev_attrs.srq_db_bar_off & (ZXDH_HW_64K_PAGE_SIZE - 1)) : ++ (iwvctx->dev_attrs.srq_db_bar_off & (ZXDH_HW_PAGE_SIZE - 1)); ++ info->base_srqn = resp.base_srqn; ++ ++ /* 防止 (srq_id - base_srqn) 发生下溢 */ ++ if (iwusrq->srq_id < info->base_srqn) { ++ zxdh_dbg(ZXDH_DBG_SRQ, ++ "%s invalid srq_id:%d < base_srqn:%d\n", ++ __func__, iwusrq->srq_id, info->base_srqn); ++ /* 已成功创建 SRQ，此处需清理 */ ++ ibv_cmd_destroy_srq(&iwusrq->ibv_srq); ++ return -EINVAL; ++ } ++ ++ info->srq_db_base = ++ (__le64 *)((__u8 *)iwvctx->srq_db + addr_offset + ++ (iwusrq->srq_id - info->base_srqn) * 8); ++ ++ set_64bit_val(info->srq_db_base, 0, ZXDH_SRQ_DB_INIT_VALUE); ++ zxdh_dbg(ZXDH_DBG_SRQ, ++ "%s info->srq_id:%d info->base_srqn:%d srq_db:0x%p\n", ++ __func__, info->srq_id, info->base_srqn, ++ iwvctx->srq_db); ++ } + zxdh_dbg( +- verbs_get_ctx(iwusrq->ibv_srq.context), ZXDH_DBG_SRQ, +- "%s info->srq_id:%d info->srq_size:%d info->srq_list_size:%d\n", +- __func__, info->srq_id, info->srq_size, info->srq_list_size); ++ ZXDH_DBG_SRQ, ++ "%s info->srq_id:%d info->srq_size:%d info->srq_list_size:%d srq_db_base:0x%p\n", ++ __func__, info->srq_id, info->srq_size, info->srq_list_size, ++ info->srq_db_base); + + return 0; + } +@@ -2790,7 +3196,7 @@ + */ + static int zxdh_vmapped_srq(struct zxdh_usrq *iwusrq, struct ibv_pd *pd, + struct ibv_srq_init_attr *attr, int srqdepth, +- struct zxdh_srq_init_info *info) ++ __u8 srqshift, struct zxdh_srq_init_info *info) + { + size_t total_srq_size; + size_t srq_pages = 0; +@@ -2798,25 +3204,27 @@ + int ret; + + total_srq_size = +- zxdh_get_total_srq_size(iwusrq, srqdepth, info->srq_size); ++ zxdh_get_total_srq_size(iwusrq, srqdepth, info->srq_size, srqshift); + srq_pages = iwusrq->buf_size >> ZXDH_HW_PAGE_SHIFT; + srq_list_pages = iwusrq->list_buf_size >> ZXDH_HW_PAGE_SHIFT; + ret = zxdh_alloc_srq_buf(iwusrq, info, total_srq_size); + if (ret) + return -ENOMEM; +- zxdh_dbg(verbs_get_ctx(iwusrq->ibv_srq.context), ZXDH_DBG_SRQ, +- "%s srq_pages:%ld srq_list_pages:%ld\n", __func__, srq_pages, +- srq_list_pages); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s srq_pages:%ld srq_list_pages:%ld\n", ++ __func__, srq_pages, srq_list_pages); + + ret = zxdh_reg_srq_mr(pd, info, total_srq_size, srq_pages, + srq_list_pages, iwusrq); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s %d ret:%d\n", __func__, __LINE__, ret); + if (ret) { + errno = ret; + goto err_dereg_srq_mr; + } + ret = create_srq(pd, iwusrq, attr, info); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s %d ret:%d\n", __func__, __LINE__, ret); + if (ret) + goto err_srq; ++ + return 0; + err_srq: + ibv_cmd_dereg_mr(&iwusrq->vmr); +@@ -2859,7 +3267,7 @@ + struct zxdh_srq_init_info *info, + struct zxdh_dev_attrs *dev_attrs) + { +- info->srq_size = srqdepth >> srqshift; ++ info->srq_size = srqdepth; + iwusrq->max_wr = info->srq_size; + iwusrq->max_sge = srq_init_attr->attr.max_sge; + iwusrq->srq_limit = srq_init_attr->attr.srq_limit; +@@ -2891,18 +3299,24 @@ + int status; + int ret; + struct zxdh_usrq *iwusrq; ++ __u32 usr_def_max_wr; + + iwvctx = container_of(pd->context, struct zxdh_uvcontext, + ibv_ctx.context); + dev_attrs = &iwvctx->dev_attrs; + + if ((zxdh_check_srq_init_attr(srq_init_attr, dev_attrs)) != 0) { +- verbs_err(&iwvctx->ibv_ctx, +- "zxdh_check_srq_init_attr failed\n"); ++ zxdh_dbg(ZXDH_DBG_SRQ, "zxdh_check_srq_init_attr failed\n"); + errno = EINVAL; + return NULL; + } + ++ if (iwvctx->is_qp_ext_mem_support && srq_init_attr->attr.max_sge > dev_attrs->max_hw_rq_sges) { ++ zxdh_dbg(ZXDH_DBG_SRQ, "zxdh_check_srq_init_attr failed, when use_blue_flame=true, srq max sge is 7.\n"); ++ errno = EINVAL; ++ return NULL; ++ } ++ + /* get shift count for maximum wqe size */ + zxdh_get_srq_wqe_shift(dev_attrs, srq_init_attr->attr.max_sge, + &srqshift); +@@ -2911,13 +3325,14 @@ + status = zxdh_get_srqdepth(dev_attrs->max_hw_srq_quanta, + srq_init_attr->attr.max_wr, srqshift, + &srqdepth); ++ usr_def_max_wr = srq_init_attr->attr.max_wr; + zxdh_dbg( +- &iwvctx->ibv_ctx, ZXDH_DBG_SRQ, ++ ZXDH_DBG_SRQ, + "%s %d status:%d srqshift:%d srqdepth:%d dev_attrs->max_hw_srq_quanta:%d srq_init_attr->attr.max_wr:%d\n", + __func__, __LINE__, status, srqshift, srqdepth, + dev_attrs->max_hw_srq_quanta, srq_init_attr->attr.max_wr); + if (status != 0) { +- verbs_err(&iwvctx->ibv_ctx, "zxdh_get_srqdepth failed\n"); ++ zxdh_dbg(ZXDH_DBG_SRQ, "zxdh_get_srqdepth failed\n"); + errno = EINVAL; + return NULL; + } +@@ -2930,29 +3345,31 @@ + + if (zxdh_init_iwusrq(iwusrq, srq_init_attr, srqdepth, srqshift, &info, + dev_attrs)) { +- verbs_err(&iwvctx->ibv_ctx, "calloc srq_wrid_array failed\n"); ++ zxdh_dbg(ZXDH_DBG_SRQ, "calloc srq_wrid_array failed\n"); + goto err_srq_wrid_array; + } +- status = zxdh_vmapped_srq(iwusrq, pd, srq_init_attr, srqdepth, &info); ++ iwusrq->ibv_srq.context = pd->context; ++ status = zxdh_vmapped_srq(iwusrq, pd, srq_init_attr, srqdepth, srqshift, &info); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s %d status:%d\n", __func__, __LINE__, status); + if (status) { +- verbs_err(&iwvctx->ibv_ctx, "zxdh_vmapped_srq failed\n"); ++ zxdh_dbg(ZXDH_DBG_SRQ, "zxdh_vmapped_srq failed\n"); + errno = status; + goto err_vmapped_srq; + } + + status = zxdh_srq_init(&iwusrq->srq, &info); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s %d status:%d\n", __func__, __LINE__, status); + if (status) { +- verbs_err(&iwvctx->ibv_ctx, "zxdh_srq_init failed\n"); ++ zxdh_dbg(ZXDH_DBG_SRQ, "zxdh_srq_init failed\n"); + errno = EINVAL; + goto err_free_srq_init; + } + zxdh_srq_wqe_init(iwusrq); + +- srq_init_attr->attr.max_wr = (srqdepth - ZXDH_SRQ_RSVD) >> srqshift; +- +- zxdh_dbg(&iwvctx->ibv_ctx, ZXDH_DBG_SRQ, +- "iwusrq->srq_id:%d info.srq_size:%d\n", iwusrq->srq_id, +- info.srq_size); ++ //srq_init_attr->attr.max_wr = (srqdepth - ZXDH_SRQ_RSVD) >> srqshift; ++ srq_init_attr->attr.max_wr = usr_def_max_wr; ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s iwusrq->srq_id:%d info.srq_size:%d\n", ++ __func__, iwusrq->srq_id, info.srq_size); + return &iwusrq->ibv_srq; + + err_free_srq_init: +@@ -2966,6 +3383,7 @@ + errno = EINVAL; + err_free_srq: + free(iwusrq); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s %d\n", __func__, __LINE__); + return NULL; + } + +@@ -2986,8 +3404,8 @@ + ret = zxdh_destroy_vmapped_srq(iwusrq); + if (ret) + goto err; +- zxdh_dbg(verbs_get_ctx(iwusrq->ibv_srq.context), ZXDH_DBG_SRQ, +- "iwusrq->srq_id:%d\n", iwusrq->srq_id); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s iwusrq->srq_id:%d\n", __func__, ++ iwusrq->srq_id); + zxdh_free_hw_buf(iwusrq->srq.srq_base, iwusrq->total_buf_size); + free(iwusrq->srq.srq_wrid_array); + free(iwusrq); +@@ -3014,9 +3432,8 @@ + sizeof(cmd)); + if (ret == 0) + iwusrq->srq_limit = srq_attr->srq_limit; +- zxdh_dbg(verbs_get_ctx(iwusrq->ibv_srq.context), ZXDH_DBG_SRQ, +- "iwusrq->srq_id:%d srq_attr->srq_limit:%d\n", iwusrq->srq_id, +- srq_attr->srq_limit); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s iwusrq->srq_id:%d srq_attr->srq_limit:%d\n", ++ __func__, iwusrq->srq_id, srq_attr->srq_limit); + return ret; + } + +@@ -3036,10 +3453,10 @@ + struct zxdh_usrq *iwusrq, struct zxdh_srq *srq) + { + if (unlikely(recv_wr->num_sge > iwusrq->max_sge)) +- return -EINVAL; ++ return ZXDH_ERR_INVALID_FRAG_COUNT; + + if (unlikely(srq->srq_ring.head == srq->srq_ring.tail)) +- return -ENOMEM; ++ return ZXDH_ERR_RING_FULL; + + return 0; + } +@@ -3080,6 +3497,13 @@ + + udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */ + set_64bit_val(wqe_64, 0, hdr); ++ ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s wqe_64[0]:0x%llx\n", __func__, wqe_64[0]); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s wqe_64[1]:0x%llx\n", __func__, wqe_64[1]); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s wqe_64[2]:0x%llx\n", __func__, wqe_64[2]); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s wqe_64[3]:0x%llx\n", __func__, wqe_64[3]); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s wqe_64[4]:0x%llx\n", __func__, wqe_64[4]); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s wqe_64[5]:0x%llx\n", __func__, wqe_64[5]); + } + + static void zxdh_get_wqe_index(struct zxdh_srq *srq, __le16 *wqe_16, __u16 *buf, +@@ -3102,6 +3526,7 @@ + + udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */ + set_64bit_val(iwusrq->srq.srq_db_base, 0, hdr); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s srq_db_base(hdr):0x%llx\n", __func__, hdr); + } + + /** +@@ -3119,6 +3544,7 @@ + __le64 *wqe_64; + __u64 temp_val; + int err = 0; ++ errno = 0; + int nreq; + __u16 *buf; + size_t buf_size; +@@ -3130,16 +3556,16 @@ + buf_size = iwusrq->max_wr * sizeof(__u16); + buf = malloc(buf_size); + if (buf == NULL) { +- verbs_err(verbs_get_ctx(iwusrq->ibv_srq.context), +- "malloc buf_size failed\n"); +- err = -ENOMEM; ++ zxdh_status_to_errno_log(ZXDH_ERR_NO_MEMORY,"srq malloc buf_size failed, srq_id=%d", iwusrq->srq_id); + goto out; + } + + for (nreq = 0; recv_wr; nreq++, recv_wr = recv_wr->next) { + err = zxdh_check_srq_valid(recv_wr, iwusrq, hw_srq); +- if (err) ++ if (err){ ++ zxdh_status_to_errno_log(err,"srq check valid failed, srq_id=%d", iwusrq->srq_id); + break; ++ } + + iwusrq->srq.srq_wrid_array[hw_srq->srq_ring.head] = + recv_wr->wr_id; +@@ -3151,15 +3577,15 @@ + zxdh_fill_srq_wqe(iwusrq, hw_srq, wqe_64, recv_wr); + } + +- zxdh_dbg(verbs_get_ctx(iwusrq->ibv_srq.context), ZXDH_DBG_SRQ, +- "nreq:%d err:%d iwusrq->srq_id:%d\n", nreq, err, +- iwusrq->srq_id); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s nreq:%d err:%d iwusrq->srq_id:%d\n", ++ __func__, nreq, err, iwusrq->srq_id); + +- if (err == 0) { ++ if (errno == 0) { + zxdh_get_wqe_index(hw_srq, wqe_16, buf, nreq, &idx); + zxdh_update_srq_db_base(iwusrq, idx); + } + out: ++ err = errno; + pthread_spin_unlock(&iwusrq->lock); + if (err) + *bad_recv_wr = recv_wr; +@@ -3191,3 +3617,10 @@ + if (env) + zxdh_debug_mask = strtol(env, NULL, 0); + } ++ ++int zxdh_get_write_imm_split_switch(void) ++{ ++ char *env; ++ env = getenv("ZXDH_WRITE_IMM_SPILT_ENABLE"); ++ return (env != NULL) ? atoi(env) : 0; ++} +diff -ruN baseline/providers/zrdma/zxdh_verbs.c.tmp b/providers/zrdma/zxdh_verbs.c.tmp +--- a/providers/zrdma/zxdh_verbs.c.tmp 1970-01-01 08:00:00.000000000 +0800 ++++ b/providers/zrdma/zxdh_verbs.c.tmp 2026-08-06 15:29:36.743641099 +0800 +@@ -0,0 +1,3655 @@ ++// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB ++/* Copyright (c) 2024 ZTE Corporation. All rights reserved. */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "zxdh_zrdma.h" ++#include "zxdh_abi.h" ++#include "wqe_rate_limit.h" ++#include "zxdh_dv.h" ++extern unsigned int rate_limit_flag[MAX_VHCA_NUM]; ++ ++uint32_t zxdh_debug_mask; ++static inline void print_fw_ver(uint64_t fw_ver, char *str, size_t len) ++{ ++ uint16_t major, minor, sub_minor, sub_major; ++ ++ major = (fw_ver >> 48) & 0xffff; ++ sub_major = (fw_ver >> 32) & 0xffff; ++ minor = (fw_ver >> 16) & 0xffff; ++ sub_minor = fw_ver & 0xffff; ++ snprintf(str, len, "%d.%02d.%02d.%02d", major, sub_major, minor, ++ sub_minor); ++} ++ ++/** ++ * zxdh_get_inline_data - get inline_multi_sge data ++ * @inline_data: uint8_t* ++ * @ib_wr: work request ptr ++ * @len: sge total length ++ */ ++static int zxdh_get_inline_data(uint8_t *inline_data, struct ibv_send_wr *ib_wr, ++ __u32 *len) ++{ ++ int num = 0; ++ int offset = 0; ++ ++ while (num < ib_wr->num_sge) { ++ *len += ib_wr->sg_list[num].length; ++ if (*len > ZXDH_MAX_INLINE_DATA_SIZE) { ++ return ZXDH_ERR_INVALID_INLINE_DATA_SIZE; ++ } ++ memcpy(inline_data + offset, ++ (void *)(uintptr_t)ib_wr->sg_list[num].addr, ++ ib_wr->sg_list[num].length); ++ offset += ib_wr->sg_list[num].length; ++ num++; ++ } ++ return 0; ++} ++ ++/** ++ * zxdh_uquery_device_ex - query device attributes including extended properties ++ * @context: user context for the device ++ * @input: extensible input struct for ibv_query_device_ex verb ++ * @attr: extended device attribute struct ++ * @attr_size: size of extended device attribute struct ++ **/ ++int zxdh_uquery_device_ex(struct ibv_context *context, ++ const struct ibv_query_device_ex_input *input, ++ struct ibv_device_attr_ex *attr, size_t attr_size) ++{ ++ struct ib_uverbs_ex_query_device_resp resp = {}; ++ size_t resp_size = sizeof(resp); ++ int ret; ++#if IBVERBS_PABI_VERSION <= 21 ++ struct ibv_query_device_ex cmd = {}; ++ uint64_t fw_ver; ++ ret = ibv_cmd_query_device_ex(context, input, attr, attr_size, ++ (uint64_t *)&fw_ver, &cmd, sizeof(cmd), ++ sizeof(cmd), &resp, resp_size, resp_size); ++ if (ret) ++ return ret; ++ print_fw_ver(fw_ver, attr->orig_attr.fw_ver, ++ sizeof(attr->orig_attr.fw_ver)); ++#elif IBVERBS_PABI_VERSION == 25 || IBVERBS_PABI_VERSION == 22 ++ struct ibv_query_device_ex cmd = {}; ++ uint64_t fw_ver; ++ ret = ibv_cmd_query_device_ex(context, input, attr, attr_size, ++ (uint64_t *)&fw_ver, &cmd, sizeof(cmd), ++ &resp, resp_size); ++ if (ret) ++ return ret; ++ print_fw_ver(fw_ver, attr->orig_attr.fw_ver, ++ sizeof(attr->orig_attr.fw_ver)); ++#else ++ ret = ibv_cmd_query_device_any(context, input, attr, attr_size, &resp, ++ &resp_size); ++ if (ret) ++ return ret; ++ ++ print_fw_ver(resp.base.fw_ver, attr->orig_attr.fw_ver, ++ sizeof(attr->orig_attr.fw_ver)); ++#endif ++ return 0; ++} ++ ++#if IBVERBS_PABI_VERSION == 25 ++int zxdh_uquery_device(struct ibv_context *context, ++ struct ibv_device_attr *dev_attr) ++{ ++ struct ibv_query_device cmd; ++ uint64_t fw_ver; ++ int status; ++ ++ memset(dev_attr, 0, sizeof(struct ibv_device_attr)); ++ status = ibv_cmd_query_device(context, dev_attr, &fw_ver, &cmd, ++ sizeof(cmd)); ++ if (status) ++ return status; ++ print_fw_ver(fw_ver, dev_attr->fw_ver, sizeof(dev_attr->fw_ver)); ++ return 0; ++} ++#endif ++ ++/** ++ * zxdh_uquery_port - get port attributes (msg size, lnk, mtu...) ++ * @context: user context of the device ++ * @port: port for the attributes ++ * @attr: to return port attributes ++ **/ ++int zxdh_uquery_port(struct ibv_context *context, uint8_t port, ++ struct ibv_port_attr *attr) ++{ ++ struct ibv_query_port cmd; ++ ++ return ibv_cmd_query_port(context, port, attr, &cmd, sizeof(cmd)); ++} ++ ++/** ++ * zxdh_ualloc_pd - allocates protection domain and return pd ptr ++ * @context: user context of the device ++ **/ ++struct ibv_pd *zxdh_ualloc_pd(struct ibv_context *context) ++{ ++ struct ibv_alloc_pd cmd; ++ struct zxdh_ualloc_pd_resp resp = {}; ++ struct zxdh_upd *iwupd; ++ int err; ++ ++ iwupd = malloc(sizeof(*iwupd)); ++ if (!iwupd) ++ return NULL; ++ ++ err = ibv_cmd_alloc_pd(context, &iwupd->ibv_pd, &cmd, sizeof(cmd), ++ &resp.ibv_resp, sizeof(resp)); ++ if (err) ++ goto err_free; ++ ++ iwupd->pd_id = resp.pd_id; ++ ++ return &iwupd->ibv_pd; ++ ++err_free: ++ free(iwupd); ++ errno = err; ++ return NULL; ++} ++ ++/** ++ * zxdh_ufree_pd - free pd resources ++ * @pd: pd to free resources ++ */ ++int zxdh_ufree_pd(struct ibv_pd *pd) ++{ ++ struct zxdh_upd *iwupd; ++ int ret; ++ ++ iwupd = container_of(pd, struct zxdh_upd, ibv_pd); ++ ret = ibv_cmd_dealloc_pd(pd); ++ if (ret) ++ return ret; ++ ++ free(iwupd); ++ ++ return 0; ++} ++ ++/** ++ * zxdh_ureg_mr - register user memory region ++ * @pd: pd for the mr ++ * @addr: user address of the memory region ++ * @length: length of the memory ++ * @hca_va: hca_va ++ * @access: access allowed on this mr ++ */ ++ struct ibv_mr *zxdh_ureg_mr(struct ibv_pd *pd, void *addr, size_t length, ++ uint64_t hca_va, int access) ++{ ++ struct zxdh_umr *umr; ++ struct zxdh_ureg_mr cmd; ++ struct zxdh_ureg_mr_resp resp = {}; ++ int err; ++ ++ umr = malloc(sizeof(*umr)); ++ if (!umr) ++ return NULL; ++ ++ cmd.reg_type = ZXDH_MEMREG_TYPE_MEM; ++ err = ibv_cmd_reg_mr(pd, addr, length, hca_va, access, &umr->vmr, ++ &cmd.ibv_cmd, sizeof(cmd), &resp.ibv_resp, ++ sizeof(resp)); ++ if (err) { ++ free(umr); ++ errno = err; ++ return NULL; ++ } ++ umr->acc_flags = access; ++ umr->host_page_size = resp.host_page_size; ++ umr->leaf_pbl_size = resp.leaf_pbl_size; ++ umr->mr_pa_pble_index = resp.mr_pa_hig; ++ umr->mr_pa_pble_index = (umr->mr_pa_pble_index << 32) | resp.mr_pa_low; ++ ++ return &umr->vmr.ibv_mr; ++} ++ ++/** ++ * zxdh_ureg_mr_dmabuf - register dma-buf based user memory region ++ * @pd: pd for the mr ++ * @offset: offset within the dma-buf ++ * @length: length of the memory region ++ * @iova: I/O Virtual Address (IOVA) for HCA access ++ * @fd: dma-buf file descriptor ++ * @access: access allowed on this mr ++ */ ++struct ibv_mr *zxdh_ureg_mr_dmabuf(struct ibv_pd *pd, uint64_t offset, ++ size_t length, uint64_t iova, int fd, ++ int access) ++{ ++ struct zxdh_umr *umr; ++ int err; ++ ++ umr = calloc(1, sizeof(*umr)); ++ if (!umr) ++ return NULL; ++ err = ibv_cmd_reg_dmabuf_mr(pd, offset, length, iova, fd, access, ++ &umr->vmr); ++ if (err) { ++ free(umr); ++ errno = err; ++ return NULL; ++ } ++ ++ return &umr->vmr.ibv_mr; ++} ++ ++/* ++ * zxdh_urereg_mr - re-register memory region ++ * @vmr: mr that was allocated ++ * @flags: bit mask to indicate which of the attr's of MR modified ++ * @pd: pd of the mr ++ * @addr: user address of the memory region ++ * @length: length of the memory ++ * @access: access allowed on this mr ++ */ ++int zxdh_urereg_mr(struct verbs_mr *vmr, int flags, struct ibv_pd *pd, ++ void *addr, size_t length, int access) ++{ ++ struct zxdh_urereg_mr cmd = {}; ++ struct ib_uverbs_rereg_mr_resp resp; ++ ++ cmd.reg_type = ZXDH_MEMREG_TYPE_MEM; ++ return ibv_cmd_rereg_mr(vmr, flags, addr, length, (uintptr_t)addr, ++ access, pd, &cmd.ibv_cmd, sizeof(cmd), &resp, ++ sizeof(resp)); ++} ++ ++/** ++ * zxdh_udereg_mr - re-register memory region ++ * @vmr: mr that was allocated ++ */ ++int zxdh_udereg_mr(struct verbs_mr *vmr) ++{ ++ int ret; ++ ++ ret = ibv_cmd_dereg_mr(vmr); ++ if (ret) ++ return ret; ++ ++ free(vmr); ++ ++ return 0; ++} ++ ++/** ++ * zxdh_ualloc_mw - allocate memory window ++ * @pd: protection domain ++ * @type: memory window type ++ */ ++struct ibv_mw *zxdh_ualloc_mw(struct ibv_pd *pd, enum ibv_mw_type type) ++{ ++ struct ibv_mw *mw; ++ struct ibv_alloc_mw cmd; ++ struct ib_uverbs_alloc_mw_resp resp; ++ ++ mw = calloc(1, sizeof(*mw)); ++ if (!mw) ++ return NULL; ++ ++ if (ibv_cmd_alloc_mw(pd, type, mw, &cmd, sizeof(cmd), &resp, ++ sizeof(resp))) { ++ free(mw); ++ return NULL; ++ } ++ ++ return mw; ++} ++ ++/** ++ * zxdh_ubind_mw - bind a memory window ++ * @qp: qp to post WR ++ * @mw: memory window to bind ++ * @mw_bind: bind info ++ */ ++int zxdh_ubind_mw(struct ibv_qp *qp, struct ibv_mw *mw, ++ struct ibv_mw_bind *mw_bind) ++{ ++ struct ibv_mw_bind_info *bind_info = &mw_bind->bind_info; ++ struct verbs_mr *vmr = verbs_get_mr(bind_info->mr); ++ struct zxdh_umr *umr = container_of(vmr, struct zxdh_umr, vmr); ++ struct ibv_send_wr wr = {}; ++ struct ibv_send_wr *bad_wr; ++ int err; ++ ++ if (vmr->mr_type != IBV_MR_TYPE_MR) ++ return -ENOTSUP; ++ ++ if (umr->acc_flags & IBV_ACCESS_ZERO_BASED) ++ return -EINVAL; ++ ++ if (mw->type != IBV_MW_TYPE_1) ++ return -EINVAL; ++ ++ wr.opcode = IBV_WR_BIND_MW; ++ wr.bind_mw.bind_info = mw_bind->bind_info; ++ wr.bind_mw.mw = mw; ++ wr.bind_mw.rkey = ibv_inc_rkey(mw->rkey); ++ ++ wr.wr_id = mw_bind->wr_id; ++ wr.send_flags = mw_bind->send_flags; ++ ++ err = zxdh_upost_send(qp, &wr, &bad_wr); ++ if (!err) ++ mw->rkey = wr.bind_mw.rkey; ++ ++ return err; ++} ++ ++/** ++ * zxdh_udealloc_mw - deallocate memory window ++ * @mw: memory window to dealloc ++ */ ++int zxdh_udealloc_mw(struct ibv_mw *mw) ++{ ++ int ret; ++ ret = ibv_cmd_dealloc_mw(mw); ++ if (ret) ++ return ret; ++ free(mw); ++ ++ return 0; ++} ++ ++int zxdh_rdmanl_saw_err_cb(struct sockaddr_nl *nla, struct nlmsgerr *nlerr, ++ void *arg) ++{ ++ bool *failed = arg; ++ ++ *failed = true; ++ return 0; ++} ++ ++struct nl_sock *zxdh_rdmanl_socket_alloc(void) ++{ ++ struct nl_sock *nl; ++ ++ nl = nl_socket_alloc(); ++ if (!nl) ++ return NULL; ++ nl_socket_disable_auto_ack(nl); ++ nl_socket_disable_msg_peek(nl); ++ ++ if (nl_connect(nl, NETLINK_RDMA)) { ++ nl_socket_free(nl); ++ return NULL; ++ } ++ return nl; ++} ++int zxdh_rdmanl_get_copy_on_fork(struct nl_sock *nl, ++ nl_recvmsg_msg_cb_t cb_func, void *data) ++{ ++ bool failed = false; ++ int ret; ++ ++ if (nl_send_simple( ++ nl, RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_SYS_GET), ++ 0, NULL, 0) < 0) ++ return -1; ++ ++ if (nl_socket_modify_err_cb(nl, NL_CB_CUSTOM, zxdh_rdmanl_saw_err_cb, ++ &failed)) ++ return -1; ++ if (nl_socket_modify_cb(nl, NL_CB_VALID, NL_CB_CUSTOM, cb_func, data)) ++ return -1; ++ do { ++ ret = nl_recvmsgs_default(nl); ++ } while (ret > 0); ++ nl_socket_modify_err_cb(nl, NL_CB_CUSTOM, NULL, NULL); ++ ++ if (ret || failed) ++ return -1; ++ return 0; ++} ++ ++#define ZXDH_RDMA_NLDEV_SYS_ATTR_COPY_ON_FORK 93 ++#define ZXDH_RDMA_NLDEV_ATTR_MAX 99 ++ ++struct nla_policy zxdh_rdmanl_policy[ZXDH_RDMA_NLDEV_ATTR_MAX] = { ++ [ZXDH_RDMA_NLDEV_SYS_ATTR_COPY_ON_FORK] = { .type = NLA_U8 }, ++}; ++ ++int zxdh_get_copy_on_fork_cb(struct nl_msg *msg, void *data) ++{ ++ struct nlattr *tb[ZXDH_RDMA_NLDEV_ATTR_MAX]; ++ int ret; ++ ++ ret = nlmsg_parse(nlmsg_hdr(msg), 0, tb, ZXDH_RDMA_NLDEV_ATTR_MAX - 1, ++ zxdh_rdmanl_policy); ++ if (ret < 0) ++ return ret; ++ ++ /* Older kernels don't support COF and don't report it through nl */ ++ if (!tb[ZXDH_RDMA_NLDEV_SYS_ATTR_COPY_ON_FORK]) { ++ *(uint8_t *)data = 0; ++ return NL_OK; ++ } ++ ++ *(uint8_t *)data = ++ nla_get_u8(tb[ZXDH_RDMA_NLDEV_SYS_ATTR_COPY_ON_FORK]); ++ return NL_OK; ++} ++ ++bool zxdh_get_copy_on_fork(void) ++{ ++ struct nl_sock *nl; ++ uint8_t cof; ++ ++ nl = zxdh_rdmanl_socket_alloc(); ++ if (!nl) ++ return false; ++ ++ if (zxdh_rdmanl_get_copy_on_fork(nl, zxdh_get_copy_on_fork_cb, &cof)) ++ cof = false; ++ ++ nl_socket_free(nl); ++ return cof; ++} ++ ++bool zxdh_get_fork_safety(void) ++{ ++ static int is_fork_safed = 0xff; ++ if (is_fork_safed == 0xff) { ++ is_fork_safed = zxdh_get_copy_on_fork(); ++ } ++ return is_fork_safed != 0; ++} ++ ++static inline unsigned long zxdh_align_up(unsigned long val, unsigned long align_val) ++{ ++ return (val + align_val - 1) & ~(align_val - 1); ++} ++ ++void *zxdh_alloc_hw_buf(size_t size) ++{ ++ void *buf; ++ size_t al_size; ++ ++ __u64 page_size = ++ (ZXDH_SYS_PAGE_SIZE == ZXDH_HW_64K_PAGE_SIZE) ? ++ ZXDH_HW_64K_PAGE_SIZE : ++ ZXDH_HW_PAGE_SIZE; ++ al_size = zxdh_align_up((unsigned long)size, (unsigned long)page_size); ++ buf = memalign(page_size, al_size); ++ if (!buf) ++ return NULL; ++ ++ /* 禁用透明巨页以避免内存碎页和性能问题 */ ++ madvise(buf, al_size, MADV_NOHUGEPAGE); ++ ++ if (zxdh_get_fork_safety()) ++ return buf; ++ if (ibv_dontfork_range(buf, al_size)) { ++ free(buf); ++ return NULL; ++ } ++ ++ return buf; ++} ++ ++static void zxdh_free_hw_buf(void *buf, size_t size) ++{ ++ size_t al_size; ++ ++ __u64 page_size = ++ (ZXDH_SYS_PAGE_SIZE == ZXDH_HW_64K_PAGE_SIZE) ? ++ ZXDH_HW_64K_PAGE_SIZE : ++ ZXDH_HW_PAGE_SIZE; ++ al_size = zxdh_align_up((unsigned long)size, (unsigned long)page_size); ++ ibv_dofork_range(buf, al_size); ++ free(buf); ++} ++ ++/** ++ * get_cq_size - returns actual cqe needed by HW ++ * @ncqe: minimum cqes requested by application ++ */ ++static inline int get_cq_size(int ncqe) ++{ ++ if (ncqe < ZXDH_U_MINCQ_SIZE) ++ ncqe = ZXDH_U_MINCQ_SIZE; ++ return ncqe; ++} ++ ++static inline size_t get_cq_total_bytes(__u32 cq_size) ++{ ++ return roundup(cq_size * sizeof(struct zxdh_cqe), ZXDH_HW_PAGE_SIZE); ++} ++ ++/** ++ * ucreate_cq - zxdh util function to create a CQ ++ * @context: ibv context ++ * @attr_ex: CQ init attributes ++ * @ext_cq: flag to create an extendable or normal CQ ++ */ ++static struct ibv_cq_ex *ucreate_cq(struct ibv_context *context, ++ struct ibv_cq_init_attr_ex *attr_ex, ++ bool ext_cq) ++{ ++ struct zxdh_cq_init_info info = {}; ++ struct zxdh_ureg_mr reg_mr_cmd = {}; ++ struct zxdh_ucreate_cq_ex cmd = {}; ++ struct zxdh_ucreate_cq_ex_resp resp = {}; ++ struct ib_uverbs_reg_mr_resp reg_mr_resp = {}; ++ struct zxdh_ureg_mr reg_mr_shadow_cmd = {}; ++ struct ib_uverbs_reg_mr_resp reg_mr_shadow_resp = {}; ++ struct zxdh_dev_attrs *dev_attrs; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_ucq *iwucq; ++ size_t total_size; ++ __u32 cq_pages; ++ int ret, ncqe; ++ __u64 resize_supported; ++ __u64 addr_offset = 0; ++ ++ iwvctx = container_of(context, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ ++ if (attr_ex->cqe < ZXDH_MIN_CQ_SIZE || ++ attr_ex->cqe > dev_attrs->max_hw_cq_size) { ++ errno = EINVAL; ++ return NULL; ++ } ++ ++ info.cq_size = get_cq_size(attr_ex->cqe); ++ info.cq_size = zxdh_cq_round_up(info.cq_size); ++ if (info.cq_size > dev_attrs->max_hw_cq_size) { ++ errno = EINVAL; ++ return NULL; ++ } ++ if (info.cq_size * 2 <= dev_attrs->max_hw_cq_size) ++ info.cq_size = info.cq_size * 2; ++ /* save the cqe requested by application */ ++ ncqe = attr_ex->cqe; ++ iwucq = calloc(1, sizeof(*iwucq)); ++ if (!iwucq) ++ return NULL; ++ ++ ret = pthread_spin_init(&iwucq->lock, PTHREAD_PROCESS_PRIVATE); ++ if (ret) { ++ errno = ret; ++ free(iwucq); ++ return NULL; ++ } ++ ++ iwucq->resize_enable = false; ++ iwucq->comp_vector = attr_ex->comp_vector; ++ list_head_init(&iwucq->resize_list); ++ total_size = get_cq_total_bytes(info.cq_size); ++ cq_pages = total_size >> ZXDH_HW_PAGE_SHIFT; ++ resize_supported = dev_attrs->feature_flags & ZXDH_FEATURE_CQ_RESIZE; ++ ++ if (!resize_supported) ++ total_size = (cq_pages << ZXDH_HW_PAGE_SHIFT) + ++ ZXDH_DB_SHADOW_AREA_SIZE; ++ ++ iwucq->buf_size = total_size; ++ info.cq_base = (struct zxdh_cqe *)zxdh_alloc_hw_buf(total_size); ++ if (!info.cq_base) ++ goto err_cq_base; ++ ++ memset(info.cq_base, 0, total_size); ++ reg_mr_cmd.reg_type = ZXDH_MEMREG_TYPE_CQ; ++ reg_mr_cmd.cq_pages = cq_pages; ++ ++ ret = ibv_cmd_reg_mr(&iwvctx->iwupd->ibv_pd, info.cq_base, total_size, ++ (uintptr_t)info.cq_base, IBV_ACCESS_LOCAL_WRITE, ++ &iwucq->vmr, ®_mr_cmd.ibv_cmd, ++ sizeof(reg_mr_cmd), ®_mr_resp, ++ sizeof(reg_mr_resp)); ++ if (ret) { ++ errno = ret; ++ goto err_dereg_mr; ++ } ++ ++ iwucq->vmr.ibv_mr.pd = &iwvctx->iwupd->ibv_pd; ++ ++ if (resize_supported) { ++ info.shadow_area = (__le64 *)zxdh_alloc_hw_buf(ZXDH_DB_SHADOW_AREA_SIZE); ++ if (!info.shadow_area) ++ goto err_dereg_mr; ++ ++ memset(info.shadow_area, 0, ZXDH_DB_SHADOW_AREA_SIZE); ++ reg_mr_shadow_cmd.reg_type = ZXDH_MEMREG_TYPE_CQ; ++ reg_mr_shadow_cmd.cq_pages = 1; ++ ret = ibv_cmd_reg_mr( ++ &iwvctx->iwupd->ibv_pd, info.shadow_area, ++ ZXDH_DB_SHADOW_AREA_SIZE, (uintptr_t)info.shadow_area, ++ IBV_ACCESS_LOCAL_WRITE, &iwucq->vmr_shadow_area, ++ ®_mr_shadow_cmd.ibv_cmd, sizeof(reg_mr_shadow_cmd), ++ ®_mr_shadow_resp, sizeof(reg_mr_shadow_resp)); ++ if (ret) { ++ errno = ret; ++ goto err_dereg_shadow; ++ } ++ ++ iwucq->vmr_shadow_area.ibv_mr.pd = &iwvctx->iwupd->ibv_pd; ++ } else { ++ info.shadow_area = (__le64 *)((__u8 *)info.cq_base + ++ (cq_pages << ZXDH_HW_PAGE_SHIFT)); ++ } ++ ++ attr_ex->cqe = info.cq_size; ++ cmd.user_cq_buf = (__u64)((uintptr_t)info.cq_base); ++ cmd.user_shadow_area = (__u64)((uintptr_t)info.shadow_area); ++ ret = ibv_cmd_create_cq_ex_wrap(context, attr_ex, &iwucq->verbs_cq, &cmd.ibv_cmd, sizeof(cmd), &resp.ibv_resp, sizeof(resp)); ++ if (ret) { ++ errno = ret; ++ goto err_dereg_shadow; ++ } ++ ++ if (ext_cq) ++ zxdh_ibvcq_ex_fill_priv_funcs(iwucq, attr_ex); ++ info.cq_id = resp.cq_id; ++ /* Do not report the cqe's burned by HW */ ++ iwucq->verbs_cq.cq.cqe = ncqe; ++ ++ addr_offset = ++ (ZXDH_SYS_PAGE_SIZE == ZXDH_HW_64K_PAGE_SIZE) ? ++ (iwvctx->dev_attrs.cq_db_bar_off & (ZXDH_HW_64K_PAGE_SIZE - 1)) : ++ 0; ++ ++ info.cqe_alloc_db = ++ (__u32 *)((__u8 *)iwvctx->cq_db + ZXDH_DB_CQ_OFFSET + addr_offset); ++ zxdh_cq_init(&iwucq->cq, &info); ++ ++ return &iwucq->verbs_cq.cq_ex; ++ ++err_dereg_shadow: ++ ibv_cmd_dereg_mr(&iwucq->vmr); ++ if (iwucq->vmr_shadow_area.ibv_mr.handle) { ++ ibv_cmd_dereg_mr(&iwucq->vmr_shadow_area); ++ if (resize_supported) ++ zxdh_free_hw_buf(info.shadow_area, ++ ZXDH_DB_SHADOW_AREA_SIZE); ++ } ++err_dereg_mr: ++ zxdh_free_hw_buf(info.cq_base, total_size); ++err_cq_base: ++ pthread_spin_destroy(&iwucq->lock); ++ ++ free(iwucq); ++ ++ return NULL; ++} ++ ++struct ibv_cq *zxdh_ucreate_cq(struct ibv_context *context, int cqe, ++ struct ibv_comp_channel *channel, ++ int comp_vector) ++{ ++ struct ibv_cq_init_attr_ex attr_ex = { ++ .cqe = cqe, ++ .channel = channel, ++ .comp_vector = comp_vector, ++ }; ++ struct ibv_cq_ex *ibvcq_ex; ++ ++ ibvcq_ex = ucreate_cq(context, &attr_ex, false); ++ ++ return ibvcq_ex ? ibv_cq_ex_to_cq(ibvcq_ex) : NULL; ++} ++ ++struct ibv_cq_ex *zxdh_ucreate_cq_ex(struct ibv_context *context, ++ struct ibv_cq_init_attr_ex *attr_ex) ++{ ++ if (attr_ex->wc_flags & ~ZXDH_CQ_SUPPORTED_WC_FLAGS) { ++ errno = EOPNOTSUPP; ++ return NULL; ++ } ++ ++ return ucreate_cq(context, attr_ex, true); ++} ++ ++/** ++ * zxdh_free_cq_buf - free memory for cq buffer ++ * @cq_buf: cq buf to free ++ */ ++static void zxdh_free_cq_buf(struct zxdh_cq_buf *cq_buf) ++{ ++ ibv_cmd_dereg_mr(&cq_buf->vmr); ++ zxdh_free_hw_buf(cq_buf->cq.cq_base, ++ get_cq_total_bytes(cq_buf->cq.cq_size)); ++ free(cq_buf); ++} ++ ++/** ++ * zxdh_process_resize_list - process the cq list to remove buffers ++ * @iwucq: cq which owns the list ++ * @lcqe_buf: cq buf where the last cqe is found ++ */ ++static int zxdh_process_resize_list(struct zxdh_ucq *iwucq, ++ struct zxdh_cq_buf *lcqe_buf) ++{ ++ struct zxdh_cq_buf *cq_buf, *next; ++ int cq_cnt = 0; ++ ++ list_for_each_safe (&iwucq->resize_list, cq_buf, next, list) { ++ if (cq_buf == lcqe_buf) ++ return cq_cnt; ++ ++ list_del(&cq_buf->list); ++ zxdh_free_cq_buf(cq_buf); ++ cq_cnt++; ++ } ++ ++ return cq_cnt; ++} ++ ++/** ++ * zxdh_udestroy_cq - destroys cq ++ * @cq: ptr to cq to be destroyed ++ */ ++int zxdh_udestroy_cq(struct ibv_cq *cq) ++{ ++ struct zxdh_dev_attrs *dev_attrs; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_ucq *iwucq; ++ __u64 cq_shadow_temp; ++ int ret; ++ ++ iwucq = container_of(cq, struct zxdh_ucq, verbs_cq.cq); ++ iwvctx = container_of(cq->context, struct zxdh_uvcontext, ++ ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ ++ ret = pthread_spin_destroy(&iwucq->lock); ++ if (ret) ++ goto err; ++ ++ get_64bit_val(iwucq->cq.shadow_area, 0, &cq_shadow_temp); ++ if (!iwucq->is_ext_buf_mode) ++ zxdh_process_resize_list(iwucq, NULL); ++ ret = ibv_cmd_destroy_cq(cq); ++ if (ret) ++ goto err; ++ ++ if (iwucq->is_ext_buf_mode) { ++ free(iwucq); ++ return 0; ++ } ++ ++ ibv_cmd_dereg_mr(&iwucq->vmr); ++ zxdh_free_hw_buf(iwucq->cq.cq_base, iwucq->buf_size); ++ ++ if (dev_attrs->feature_flags & ZXDH_FEATURE_CQ_RESIZE) { ++ ibv_cmd_dereg_mr(&iwucq->vmr_shadow_area); ++ zxdh_free_hw_buf(iwucq->cq.shadow_area, ++ ZXDH_DB_SHADOW_AREA_SIZE); ++ } ++ free(iwucq); ++ return 0; ++ ++err: ++ return ret; ++} ++ ++int zxdh_umodify_cq(struct ibv_cq *cq, struct ibv_modify_cq_attr *attr) ++{ ++ struct ibv_modify_cq cmd = {}; ++ ++ return ibv_cmd_modify_cq(cq, attr, &cmd, sizeof(cmd)); ++} ++ ++static enum ibv_wc_status zxdh_err_to_ib_wc_status(__u32 opcode) ++{ ++ switch (opcode) { ++ case ZXDH_RX_WQE_LEN_ERR: ++ return IBV_WC_LOC_LEN_ERR; ++ case ZXDH_TX_ACK_SYS_TOP_VADDR_LEN_CHECK_ERR: ++ case ZXDH_TX_ACK_SYS_TOP_LKEY_CHECK_ERR: ++ case ZXDH_TX_ACK_SYS_TOP_ACCESS_RIGHT_CHECK_ERR: ++ case ZXDH_RX_MR_MW_STATE_FREE_ERR: ++ case ZXDH_RX_MR_MW_STATE_INVALID_ERR: ++ case ZXDH_RX_MR_MW_PD_CHECK_ERR: ++ case ZXDH_RX_MR_MW_KEY_CHECK_ERR: ++ case ZXDH_RX_MR_MW_STAG_INDEX_CHECK_ERR: ++ case ZXDH_RX_MR_MW_BOUNDARY_CHECK_ERR: ++ case ZXDH_RX_MR_MW_0STAG_INDEX_CHECK_ERR: ++ case ZXDH_RX_MW_STATE_INVALID_ERR: ++ case ZXDH_RX_MW_PD_CHECK_ERR: ++ case ZXDH_RX_MW_STAG_INDEX_CHECK_ERR: ++ case ZXDH_RX_MW_SHARE_MR_CHECK_ERR: ++ case ZXDH_RX_MR_PD_CHECK_ERR: ++ case ZXDH_RX_MR_SHARE_MR_CHECK_ERR: ++ case ZXDH_RX_MR_MW_ACCESS_CHECK_ERR: ++ return IBV_WC_LOC_PROT_ERR; ++ case ZXDH_TX_PARSE_TOP_WQE_FLUSH: ++ return IBV_WC_WR_FLUSH_ERR; ++ case ZXDH_TX_ACK_SYS_TOP_NAK_INVALID_REQ: ++ return IBV_WC_REM_INV_REQ_ERR; ++ case ZXDH_TX_ACK_SYS_TOP_NAK_REMOTE_ACCESS_ERR: ++ case ZXDH_RX_MW_RKEY_CHECK_ERR: ++ case ZXDH_RX_MR_RKEY_CHECK_ERR: ++ return IBV_WC_REM_ACCESS_ERR; ++ case ZXDH_TX_ACK_SYS_TOP_NAK_REMOTE_OPERATIONAL_ERR: ++ return IBV_WC_REM_OP_ERR; ++ case ZXDH_TX_ACK_SYS_TOP_NAK_RETRY_LIMIT: ++ case ZXDH_TX_ACK_SYS_TOP_READ_RETRY_LIMIT: ++ case ZXDH_TX_ACK_SYS_TOP_TIMEOUT_RETRY_LIMIT: ++ return IBV_WC_RETRY_EXC_ERR; ++ case ZXDH_TX_ACK_SYS_TOP_RNR_RETRY_LIMIT: ++ return IBV_WC_RNR_RETRY_EXC_ERR; ++ case ZXDH_TX_PARSE_TOP_AXI_ERR: ++ case ZXDH_RX_AXI_RESP_ERR: ++ return IBV_WC_FATAL_ERR; ++ default: ++ return IBV_WC_GENERAL_ERR; ++ } ++} ++ ++/** ++ * zxdh_process_cqe_ext - process current cqe for extended CQ ++ * @cur_cqe - current cqe info ++ */ ++static inline void zxdh_process_cqe_ext(struct zxdh_cq_poll_info *cur_cqe) ++{ ++ struct zxdh_ucq *iwucq = ++ container_of(cur_cqe, struct zxdh_ucq, cur_cqe); ++ struct ibv_cq_ex *ibvcq_ex = &iwucq->verbs_cq.cq_ex; ++ ++ ibvcq_ex->wr_id = cur_cqe->wr_id; ++ if (cur_cqe->error) ++ ibvcq_ex->status = zxdh_err_to_ib_wc_status( ++ cur_cqe->major_err << 16 | cur_cqe->minor_err); ++ else ++ ibvcq_ex->status = IBV_WC_SUCCESS; ++} ++ ++/** ++ * zxdh_process_cqe - process current cqe info ++ * @entry - ibv_wc object to fill in for non-extended CQ ++ * @cur_cqe - current cqe info ++ */ ++static inline void zxdh_process_cqe(struct ibv_wc *entry, ++ struct zxdh_cq_poll_info *cur_cqe) ++{ ++ struct zxdh_qp *qp; ++ struct ibv_qp *ib_qp; ++ ++ entry->wc_flags = 0; ++ entry->wr_id = cur_cqe->wr_id; ++ entry->qp_num = cur_cqe->qp_id; ++ qp = cur_cqe->qp_handle; ++ ib_qp = qp->back_qp; ++ ++ if (cur_cqe->error) { ++ entry->status = zxdh_err_to_ib_wc_status( ++ cur_cqe->major_err << 16 | cur_cqe->minor_err); ++ entry->vendor_err = ++ cur_cqe->major_err << 16 | cur_cqe->minor_err; ++ } else { ++ entry->status = IBV_WC_SUCCESS; ++ } ++ ++ if (cur_cqe->imm_valid) { ++ entry->imm_data = htonl(cur_cqe->imm_data); ++ entry->wc_flags |= IBV_WC_WITH_IMM; ++ } ++ ++ switch (cur_cqe->op_type) { ++ case ZXDH_OP_TYPE_SEND: ++ case ZXDH_OP_TYPE_SEND_WITH_IMM: ++ case ZXDH_OP_TYPE_SEND_INV: ++ case ZXDH_OP_TYPE_UD_SEND: ++ case ZXDH_OP_TYPE_UD_SEND_WITH_IMM: ++ entry->opcode = IBV_WC_SEND; ++ break; ++ case ZXDH_OP_TYPE_WRITE: ++ case ZXDH_OP_TYPE_WRITE_WITH_IMM: ++ entry->opcode = IBV_WC_RDMA_WRITE; ++ break; ++ case ZXDH_OP_TYPE_READ: ++ entry->opcode = IBV_WC_RDMA_READ; ++ break; ++ case ZXDH_OP_TYPE_BIND_MW: ++ entry->opcode = IBV_WC_BIND_MW; ++ break; ++ case ZXDH_OP_TYPE_LOCAL_INV: ++ entry->opcode = IBV_WC_LOCAL_INV; ++ break; ++ case ZXDH_OP_TYPE_REC_IMM: ++ entry->opcode = IBV_WC_RECV; ++ entry->wc_flags |= IBV_WC_WITH_IMM; ++ break; ++ case ZXDH_OP_TYPE_REC: ++ entry->opcode = IBV_WC_RECV; ++ if (ib_qp->qp_type != IBV_QPT_UD && cur_cqe->stag_invalid_set) { ++ entry->invalidated_rkey = cur_cqe->inv_stag; ++ entry->wc_flags |= IBV_WC_WITH_INV; ++ } ++ break; ++ case ZXDH_OP_TYPE_REC_WRITE_IMM: ++ entry->opcode = IBV_WC_RECV_RDMA_WITH_IMM; ++ entry->wc_flags |= IBV_WC_WITH_IMM; ++ break; ++ default: ++ entry->status = IBV_WC_GENERAL_ERR; ++ return; ++ } ++ ++ if (ib_qp->qp_type == IBV_QPT_UD) { ++ entry->src_qp = cur_cqe->ud_src_qpn; ++ entry->wc_flags |= IBV_WC_GRH; ++ entry->sl = cur_cqe->ipv4 ? 2 : 1; ++ } else { ++ entry->src_qp = cur_cqe->qp_id; ++ } ++ entry->byte_len = cur_cqe->bytes_xfered; ++} ++ ++/** ++ * zxdh_poll_one - poll one entry of the CQ ++ * @cq: cq to poll ++ * @cur_cqe: current CQE info to be filled in ++ * @entry: ibv_wc object to be filled for non-extended CQ or NULL for extended CQ ++ * ++ * Returns the internal zxdh device error code or 0 on success ++ */ ++static int zxdh_poll_one(struct zxdh_cq *cq, struct zxdh_cq_poll_info *cur_cqe, ++ struct ibv_wc *entry) ++{ ++ int ret = zxdh_cq_poll_cmpl(cq, cur_cqe); ++ ++ if (ret) ++ return ret; ++ ++ if (entry) ++ zxdh_process_cqe(entry, cur_cqe); ++ else ++ zxdh_process_cqe_ext(cur_cqe); ++ ++ return 0; ++} ++ ++/** ++ * __zxdh_upoll_resize_cq - zxdh util function to poll device CQ ++ * @iwucq: zxdh cq to poll ++ * @num_entries: max cq entries to poll ++ * @entry: pointer to array of ibv_wc objects to be filled in for each completion or NULL if ext CQ ++ * ++ * Returns non-negative value equal to the number of completions ++ * found. On failure, -EINVAL ++ */ ++static int __zxdh_upoll_resize_cq(struct zxdh_ucq *iwucq, int num_entries, ++ struct ibv_wc *entry) ++{ ++ struct zxdh_cq_buf *cq_buf, *next; ++ struct zxdh_cq_buf *last_buf = NULL; ++ struct zxdh_cq_poll_info *cur_cqe = &iwucq->cur_cqe; ++ bool cq_new_cqe = false; ++ int resized_bufs = 0; ++ int npolled = 0; ++ int ret; ++ ++ /* go through the list of previously resized CQ buffers */ ++ list_for_each_safe (&iwucq->resize_list, cq_buf, next, list) { ++ while (npolled < num_entries) { ++ ret = zxdh_poll_one(&cq_buf->cq, cur_cqe, ++ entry ? entry + npolled : NULL); ++ if (ret == ZXDH_SUCCESS) { ++ ++npolled; ++ cq_new_cqe = true; ++ continue; ++ } ++ if (ret == ZXDH_ERR_Q_EMPTY) ++ break; ++ if (ret == ZXDH_ERR_RETRY_ACK_NOT_EXCEED_ERR) ++ break; ++ /* QP using the CQ is destroyed. Skip reporting this CQE */ ++ if (ret == ZXDH_ERR_Q_DESTROYED) { ++ cq_new_cqe = true; ++ continue; ++ } ++ printf("__zrdma_upoll_cq resize goto error failed\n"); ++ goto error; ++ } ++ ++ /* save the resized CQ buffer which received the last cqe */ ++ if (cq_new_cqe) ++ last_buf = cq_buf; ++ cq_new_cqe = false; ++ } ++ ++ /* check the current CQ for new cqes */ ++ while (npolled < num_entries) { ++ ret = zxdh_poll_one(&iwucq->cq, cur_cqe, ++ entry ? entry + npolled : NULL); ++ if (ret == ZXDH_SUCCESS) { ++ ++npolled; ++ cq_new_cqe = true; ++ continue; ++ } ++ if (ret == ZXDH_ERR_Q_EMPTY) ++ break; ++ if (ret == ZXDH_ERR_RETRY_ACK_NOT_EXCEED_ERR) ++ break; ++ /* QP using the CQ is destroyed. Skip reporting this CQE */ ++ if (ret == ZXDH_ERR_Q_DESTROYED) { ++ cq_new_cqe = true; ++ continue; ++ } ++ printf("__zrdma_upoll_cq goto error failed\n"); ++ goto error; ++ } ++ if (cq_new_cqe) ++ /* all previous CQ resizes are complete */ ++ resized_bufs = zxdh_process_resize_list(iwucq, NULL); ++ else if (last_buf) ++ /* only CQ resizes up to the last_buf are complete */ ++ resized_bufs = zxdh_process_resize_list(iwucq, last_buf); ++ if (resized_bufs) ++ /* report to the HW the number of complete CQ resizes */ ++ zxdh_cq_set_resized_cnt(&iwucq->cq, resized_bufs); ++ ++ return npolled; ++ ++error: ++ ++ return -EINVAL; ++} ++ ++/** ++ * __zxdh_upoll_current_cq - zxdh util function to poll device CQ ++ * @iwucq: zxdh cq to poll ++ * @num_entries: max cq entries to poll ++ * @entry: pointer to array of ibv_wc objects to be filled in for each completion or NULL if ext CQ ++ * ++ * Returns non-negative value equal to the number of completions ++ * found. On failure, -EINVAL ++ */ ++static int __zxdh_upoll_curent_cq(struct zxdh_ucq *iwucq, int num_entries, ++ struct ibv_wc *entry) ++{ ++ struct zxdh_cq_poll_info *cur_cqe = &iwucq->cur_cqe; ++ int npolled = 0; ++ int ret; ++ ++ /* check the current CQ for new cqes */ ++ while (npolled < num_entries) { ++ ret = zxdh_poll_one(&iwucq->cq, cur_cqe, ++ entry ? entry + npolled : NULL); ++ if (ret == ZXDH_ERR_SRQ_EMPTY || ret == ZXDH_ERR_Q_DESTROYED) ++ continue; ++ ++ if (unlikely(ret != ZXDH_SUCCESS)) ++ break; ++ ++npolled; ++ } ++ ++ return npolled; ++} ++ ++/** ++ * zxdh_upoll_cq - verb API callback to poll device CQ ++ * @cq: ibv_cq to poll ++ * @num_entries: max cq entries to poll ++ * @entry: pointer to array of ibv_wc objects to be filled in for each completion ++ * ++ * Returns non-negative value equal to the number of completions ++ * found and a negative error code on failure ++ */ ++int zxdh_upoll_cq(struct ibv_cq *cq, int num_entries, struct ibv_wc *entry) ++{ ++ struct zxdh_ucq *iwucq; ++ int ret; ++ ++ iwucq = container_of(cq, struct zxdh_ucq, verbs_cq.cq); ++ ret = pthread_spin_lock(&iwucq->lock); ++ if (ret) ++ return -ret; ++ ++ if (likely(!iwucq->resize_enable)) ++ ret = __zxdh_upoll_curent_cq(iwucq, num_entries, entry); ++ else ++ ret = __zxdh_upoll_resize_cq(iwucq, num_entries, entry); ++ ++ pthread_spin_unlock(&iwucq->lock); ++ ++ return ret; ++} ++ ++/** ++ * zxdh_start_poll - verb_ex API callback to poll batch of WC's ++ * @ibvcq_ex: ibv extended CQ ++ * @attr: attributes (not used) ++ * ++ * Start polling batch of work completions. Return 0 on success, ENONENT when ++ * no completions are available on CQ. And an error code on errors ++ */ ++static int zxdh_start_poll(struct ibv_cq_ex *ibvcq_ex, ++ struct ibv_poll_cq_attr *attr) ++{ ++ struct zxdh_ucq *iwucq; ++ int ret; ++ ++ iwucq = container_of(ibvcq_ex, struct zxdh_ucq, verbs_cq.cq_ex); ++ ret = pthread_spin_lock(&iwucq->lock); ++ if (ret) ++ return ret; ++ ++ if (!iwucq->resize_enable) { ++ ret = __zxdh_upoll_curent_cq(iwucq, 1, NULL); ++ if (ret == 1) ++ return 0; ++ } else { ++ ret = __zxdh_upoll_resize_cq(iwucq, 1, NULL); ++ if (ret == 1) ++ return 0; ++ } ++ ++ /* No Completions on CQ */ ++ if (!ret) ++ ret = ENOENT; ++ ++ pthread_spin_unlock(&iwucq->lock); ++ ++ return ret; ++} ++ ++/** ++ * zxdh_next_poll - verb_ex API callback to get next WC ++ * @ibvcq_ex: ibv extended CQ ++ * ++ * Return 0 on success, ENONENT when no completions are available on CQ. ++ * And an error code on errors ++ */ ++static int zxdh_next_poll(struct ibv_cq_ex *ibvcq_ex) ++{ ++ struct zxdh_ucq *iwucq; ++ int ret; ++ ++ iwucq = container_of(ibvcq_ex, struct zxdh_ucq, verbs_cq.cq_ex); ++ if (!iwucq->resize_enable) { ++ ret = __zxdh_upoll_curent_cq(iwucq, 1, NULL); ++ if (ret == 1) ++ return 0; ++ } else { ++ ret = __zxdh_upoll_resize_cq(iwucq, 1, NULL); ++ if (ret == 1) ++ return 0; ++ } ++ ++ /* No Completions on CQ */ ++ if (!ret) ++ ret = ENOENT; ++ ++ return ret; ++} ++ ++/** ++ * zxdh_end_poll - verb_ex API callback to end polling of WC's ++ * @ibvcq_ex: ibv extended CQ ++ */ ++static void zxdh_end_poll(struct ibv_cq_ex *ibvcq_ex) ++{ ++ struct zxdh_ucq *iwucq = ++ container_of(ibvcq_ex, struct zxdh_ucq, verbs_cq.cq_ex); ++ ++ pthread_spin_unlock(&iwucq->lock); ++} ++ ++/** ++ * zxdh_wc_read_completion_ts - Get completion timestamp ++ * @ibvcq_ex: ibv extended CQ ++ * ++ * Get completion timestamp in HCA clock units ++ */ ++static uint64_t zxdh_wc_read_completion_ts(struct ibv_cq_ex *ibvcq_ex) ++{ ++ struct zxdh_ucq *iwucq = ++ container_of(ibvcq_ex, struct zxdh_ucq, verbs_cq.cq_ex); ++#define HCA_CORE_CLOCK_800_MHZ 800 ++ ++ return iwucq->cur_cqe.tcp_seq_num_rtt / HCA_CORE_CLOCK_800_MHZ; ++} ++ ++/** ++ * zxdh_wc_read_completion_wallclock_ns - Get completion timestamp in ns ++ * @ibvcq_ex: ibv extended CQ ++ * ++ * Get completion timestamp from current completion in wall clock nanoseconds ++ */ ++static uint64_t zxdh_wc_read_completion_wallclock_ns(struct ibv_cq_ex *ibvcq_ex) ++{ ++ struct zxdh_ucq *iwucq = ++ container_of(ibvcq_ex, struct zxdh_ucq, verbs_cq.cq_ex); ++ ++ /* RTT is in usec */ ++ return (uint64_t)iwucq->cur_cqe.tcp_seq_num_rtt * 1000; ++} ++ ++static enum ibv_wc_opcode zxdh_wc_read_opcode(struct ibv_cq_ex *ibvcq_ex) ++{ ++ struct zxdh_ucq *iwucq = ++ container_of(ibvcq_ex, struct zxdh_ucq, verbs_cq.cq_ex); ++ ++ switch (iwucq->cur_cqe.op_type) { ++ case ZXDH_OP_TYPE_WRITE: ++ case ZXDH_OP_TYPE_WRITE_WITH_IMM: ++ return IBV_WC_RDMA_WRITE; ++ case ZXDH_OP_TYPE_READ: ++ return IBV_WC_RDMA_READ; ++ case ZXDH_OP_TYPE_SEND: ++ case ZXDH_OP_TYPE_SEND_WITH_IMM: ++ case ZXDH_OP_TYPE_SEND_INV: ++ case ZXDH_OP_TYPE_UD_SEND: ++ case ZXDH_OP_TYPE_UD_SEND_WITH_IMM: ++ return IBV_WC_SEND; ++ case ZXDH_OP_TYPE_BIND_MW: ++ return IBV_WC_BIND_MW; ++ case ZXDH_OP_TYPE_REC_IMM: ++ case ZXDH_OP_TYPE_REC: ++ return IBV_WC_RECV; ++ case ZXDH_OP_TYPE_REC_WRITE_IMM: ++ return IBV_WC_RECV_RDMA_WITH_IMM; ++ case ZXDH_OP_TYPE_LOCAL_INV: ++ return IBV_WC_LOCAL_INV; ++ } ++ ++ return 0; ++} ++ ++static uint32_t zxdh_wc_read_vendor_err(struct ibv_cq_ex *ibvcq_ex) ++{ ++ struct zxdh_cq_poll_info *cur_cqe; ++ struct zxdh_ucq *iwucq; ++ ++ iwucq = container_of(ibvcq_ex, struct zxdh_ucq, verbs_cq.cq_ex); ++ cur_cqe = &iwucq->cur_cqe; ++ ++ return cur_cqe->error ? cur_cqe->major_err << 16 | cur_cqe->minor_err : ++ 0; ++} ++ ++static unsigned int zxdh_wc_read_wc_flags(struct ibv_cq_ex *ibvcq_ex) ++{ ++ struct zxdh_cq_poll_info *cur_cqe; ++ struct zxdh_ucq *iwucq; ++ struct zxdh_qp *qp; ++ struct ibv_qp *ib_qp; ++ unsigned int wc_flags = 0; ++ ++ iwucq = container_of(ibvcq_ex, struct zxdh_ucq, verbs_cq.cq_ex); ++ cur_cqe = &iwucq->cur_cqe; ++ qp = cur_cqe->qp_handle; ++ ib_qp = qp->back_qp; ++ ++ if (cur_cqe->imm_valid) ++ wc_flags |= IBV_WC_WITH_IMM; ++ ++ if (ib_qp->qp_type == IBV_QPT_UD) { ++ wc_flags |= IBV_WC_GRH; ++ } else { ++ if (cur_cqe->stag_invalid_set) { ++ switch (cur_cqe->op_type) { ++ case ZXDH_OP_TYPE_REC: ++ wc_flags |= IBV_WC_WITH_INV; ++ break; ++ } ++ } ++ } ++ ++ return wc_flags; ++} ++ ++static uint32_t zxdh_wc_read_byte_len(struct ibv_cq_ex *ibvcq_ex) ++{ ++ struct zxdh_ucq *iwucq = ++ container_of(ibvcq_ex, struct zxdh_ucq, verbs_cq.cq_ex); ++ ++ return iwucq->cur_cqe.bytes_xfered; ++} ++ ++static __be32 zxdh_wc_read_imm_data(struct ibv_cq_ex *ibvcq_ex) ++{ ++ struct zxdh_cq_poll_info *cur_cqe; ++ struct zxdh_ucq *iwucq; ++ ++ iwucq = container_of(ibvcq_ex, struct zxdh_ucq, verbs_cq.cq_ex); ++ cur_cqe = &iwucq->cur_cqe; ++ ++ return cur_cqe->imm_valid ? htonl(cur_cqe->imm_data) : 0; ++} ++ ++static uint32_t zxdh_wc_read_qp_num(struct ibv_cq_ex *ibvcq_ex) ++{ ++ struct zxdh_ucq *iwucq = ++ container_of(ibvcq_ex, struct zxdh_ucq, verbs_cq.cq_ex); ++ ++ return iwucq->cur_cqe.qp_id; ++} ++ ++static uint32_t zxdh_wc_read_src_qp(struct ibv_cq_ex *ibvcq_ex) ++{ ++ struct zxdh_cq_poll_info *cur_cqe; ++ struct zxdh_ucq *iwucq; ++ struct zxdh_qp *qp; ++ struct ibv_qp *ib_qp; ++ ++ iwucq = container_of(ibvcq_ex, struct zxdh_ucq, verbs_cq.cq_ex); ++ cur_cqe = &iwucq->cur_cqe; ++ qp = cur_cqe->qp_handle; ++ ib_qp = qp->back_qp; ++ ++ return ib_qp->qp_type == IBV_QPT_UD ? cur_cqe->ud_src_qpn : ++ cur_cqe->qp_id; ++} ++ ++static uint32_t zxdh_wc_read_slid(struct ibv_cq_ex *ibvcq_ex) ++{ ++ return 0; ++} ++ ++static uint8_t zxdh_wc_read_sl(struct ibv_cq_ex *ibvcq_ex) ++{ ++ return 0; ++} ++ ++static uint8_t zxdh_wc_read_dlid_path_bits(struct ibv_cq_ex *ibvcq_ex) ++{ ++ return 0; ++} ++ ++void zxdh_ibvcq_ex_fill_priv_funcs(struct zxdh_ucq *iwucq, ++ struct ibv_cq_init_attr_ex *attr_ex) ++{ ++ struct ibv_cq_ex *ibvcq_ex = &iwucq->verbs_cq.cq_ex; ++ ++ ibvcq_ex->start_poll = zxdh_start_poll; ++ ibvcq_ex->end_poll = zxdh_end_poll; ++ ibvcq_ex->next_poll = zxdh_next_poll; ++ ++ if (attr_ex->wc_flags & IBV_WC_EX_WITH_COMPLETION_TIMESTAMP) { ++ ibvcq_ex->read_completion_ts = zxdh_wc_read_completion_ts; ++ iwucq->report_rtt = true; ++ } ++ if (attr_ex->wc_flags & IBV_WC_EX_WITH_COMPLETION_TIMESTAMP_WALLCLOCK) { ++ ibvcq_ex->read_completion_wallclock_ns = ++ zxdh_wc_read_completion_wallclock_ns; ++ iwucq->report_rtt = true; ++ } ++ ++ ibvcq_ex->read_opcode = zxdh_wc_read_opcode; ++ ibvcq_ex->read_vendor_err = zxdh_wc_read_vendor_err; ++ ibvcq_ex->read_wc_flags = zxdh_wc_read_wc_flags; ++ ++ if (attr_ex->wc_flags & IBV_WC_EX_WITH_BYTE_LEN) ++ ibvcq_ex->read_byte_len = zxdh_wc_read_byte_len; ++ if (attr_ex->wc_flags & IBV_WC_EX_WITH_IMM) ++ ibvcq_ex->read_imm_data = zxdh_wc_read_imm_data; ++ if (attr_ex->wc_flags & IBV_WC_EX_WITH_QP_NUM) ++ ibvcq_ex->read_qp_num = zxdh_wc_read_qp_num; ++ if (attr_ex->wc_flags & IBV_WC_EX_WITH_SRC_QP) ++ ibvcq_ex->read_src_qp = zxdh_wc_read_src_qp; ++ if (attr_ex->wc_flags & IBV_WC_EX_WITH_SLID) ++ ibvcq_ex->read_slid = zxdh_wc_read_slid; ++ if (attr_ex->wc_flags & IBV_WC_EX_WITH_SL) ++ ibvcq_ex->read_sl = zxdh_wc_read_sl; ++ if (attr_ex->wc_flags & IBV_WC_EX_WITH_DLID_PATH_BITS) ++ ibvcq_ex->read_dlid_path_bits = zxdh_wc_read_dlid_path_bits; ++} ++ ++/** ++ * zxdh_arm_cq - arm of cq ++ * @iwucq: cq to which arm ++ * @cq_notify: notification params ++ */ ++static void zxdh_arm_cq(struct zxdh_ucq *iwucq, enum zxdh_cmpl_notify cq_notify) ++{ ++ iwucq->is_armed = true; ++ iwucq->last_notify = cq_notify; ++ ++ zxdh_cq_request_notification(&iwucq->cq, cq_notify); ++} ++ ++/** ++ * zxdh_uarm_cq - callback for arm of cq ++ * @cq: cq to arm ++ * @solicited: to get notify params ++ */ ++int zxdh_uarm_cq(struct ibv_cq *cq, int solicited) ++{ ++ struct zxdh_ucq *iwucq; ++ enum zxdh_cmpl_notify cq_notify = ZXDH_CQ_COMPL_EVENT; ++ bool promo_event = false; ++ int ret; ++ ++ iwucq = container_of(cq, struct zxdh_ucq, verbs_cq.cq); ++ if (solicited) { ++ cq_notify = ZXDH_CQ_COMPL_SOLICITED; ++ } else { ++ if (iwucq->last_notify == ZXDH_CQ_COMPL_SOLICITED) ++ promo_event = true; ++ } ++ ++ ret = pthread_spin_lock(&iwucq->lock); ++ if (ret) ++ return ret; ++ ++ if (!iwucq->is_armed || promo_event) ++ zxdh_arm_cq(iwucq, cq_notify); ++ ++ pthread_spin_unlock(&iwucq->lock); ++ ++ return 0; ++} ++ ++/** ++ * zxdh_cq_event - cq to do completion event ++ * @cq: cq to arm ++ */ ++void zxdh_cq_event(struct ibv_cq *cq) ++{ ++ struct zxdh_ucq *iwucq; ++ ++ iwucq = container_of(cq, struct zxdh_ucq, verbs_cq.cq); ++ if (pthread_spin_lock(&iwucq->lock)) ++ return; ++ ++ iwucq->is_armed = false; ++ ++ pthread_spin_unlock(&iwucq->lock); ++} ++ ++void *zxdh_mmap(int fd, __u32 len, off_t offset) ++{ ++ __u64 mmap_len = ++ (ZXDH_SYS_PAGE_SIZE == ZXDH_HW_64K_PAGE_SIZE) ? ++ ZXDH_HW_64K_PAGE_SIZE : ++ len; ++ ++ return mmap(NULL, mmap_len, PROT_WRITE | PROT_READ, MAP_SHARED, ++ fd, offset); ++} ++ ++void zxdh_munmap(void *map, __u32 len) ++{ ++ __u64 mmap_len = ++ (ZXDH_SYS_PAGE_SIZE == ZXDH_HW_64K_PAGE_SIZE) ? ++ ZXDH_HW_64K_PAGE_SIZE : ++ len; ++ ++ if (map == NULL) { ++ return; ++ } ++ ++ ibv_dofork_range(map, mmap_len); ++ munmap(map, mmap_len); ++} ++ ++/** ++ * zxdh_destroy_vmapped_qp - destroy resources for qp ++ * @iwuqp: qp struct for resources ++ */ ++static int zxdh_destroy_vmapped_qp(struct zxdh_uqp *iwuqp) ++{ ++ int ret; ++ ++ ret = ibv_cmd_destroy_qp(&iwuqp->vqp.qp); ++ if (ret) ++ return ret; ++ if (iwuqp->ext_buf_mode == 0){ ++ ibv_cmd_dereg_mr(&iwuqp->vmr); ++ } ++ ++ return 0; ++} ++ ++/** ++ * zxdh_vmapped_qp - create resources for qp ++ * @iwuqp: qp struct for resources ++ * @pd: pd for the qp ++ * @attr: attributes of qp passed ++ * @resp: response back from create qp ++ * @sqdepth: depth of sq ++ * @rqdepth: depth of rq ++ * @info: info for initializing user level qp ++ * @abi_ver: abi version of the create qp command ++ */ ++static int zxdh_vmapped_qp(struct zxdh_uqp *iwuqp, struct ibv_pd *pd, ++ struct ibv_qp_init_attr *attr, int sqdepth, ++ int rqdepth, struct zxdh_qp_init_info *info, ++ bool legacy_mode) ++{ ++ struct zxdh_ucreate_qp cmd = {}; ++ size_t sqsize, rqsize, totalqpsize; ++ struct zxdh_ucreate_qp_resp resp = {}; ++ struct zxdh_ureg_mr reg_mr_cmd = {}; ++ struct ib_uverbs_reg_mr_resp reg_mr_resp = {}; ++ int ret; ++ ++ rqsize = 0; ++ sqsize = roundup(sqdepth * ZXDH_QP_SQE_MIN_SIZE, ZXDH_HW_PAGE_SIZE); ++ if (iwuqp->is_srq == false) { ++ rqsize = roundup((rqdepth << info->rqshift) * ZXDH_QP_RQE_MIN_SIZE, ++ ZXDH_HW_PAGE_SIZE); ++ totalqpsize = rqsize + sqsize + ZXDH_DB_SHADOW_AREA_SIZE; ++ } else { ++ totalqpsize = sqsize + ZXDH_DB_SHADOW_AREA_SIZE; ++ } ++ info->sq = (struct zxdh_qp_sq_quanta *)zxdh_alloc_hw_buf(totalqpsize); ++ iwuqp->buf_size = totalqpsize; ++ ++ if (!info->sq) ++ return -ENOMEM; ++ ++ memset(info->sq, 0, totalqpsize); ++ if (iwuqp->is_srq == false) { ++ info->rq = (struct zxdh_qp_rq_quanta *)&info ++ ->sq[sqsize / ZXDH_QP_SQE_MIN_SIZE]; ++ info->shadow_area = ++ info->rq[rqsize / ZXDH_QP_RQE_MIN_SIZE].elem; ++ reg_mr_cmd.rq_pages = rqsize >> ZXDH_HW_PAGE_SHIFT; ++ } else { ++ info->shadow_area = ++ (__le64 *)&info->sq[sqsize / ZXDH_QP_SQE_MIN_SIZE]; ++ } ++ reg_mr_cmd.reg_type = ZXDH_MEMREG_TYPE_QP; ++ reg_mr_cmd.sq_pages = sqsize >> ZXDH_HW_PAGE_SHIFT; ++ ret = ibv_cmd_reg_mr(pd, info->sq, totalqpsize, (uintptr_t)info->sq, ++ IBV_ACCESS_LOCAL_WRITE, &iwuqp->vmr, ++ ®_mr_cmd.ibv_cmd, sizeof(reg_mr_cmd), ++ ®_mr_resp, sizeof(reg_mr_resp)); ++ if (ret) ++ goto err_dereg_mr; ++ ++ cmd.user_wqe_bufs = (__u64)((uintptr_t)info->sq); ++ cmd.user_compl_ctx = (__u64)(uintptr_t)&iwuqp->qp; ++ ++ if (info->is_qp_ext_mem_support) { ++ cmd.support_ext_qp = 1; ++ } ++ ret = ibv_cmd_create_qp(pd, &iwuqp->vqp.qp, attr, &cmd.ibv_cmd, ++ sizeof(cmd), &resp.ibv_resp, ++ sizeof(struct zxdh_ucreate_qp_resp)); ++ if (ret) ++ goto err_qp; ++#if defined(__x86_64__) || defined (__i386__) ++ if (resp.wqe_rate_limit_flag) { ++ rdma_wqe_rate_limit_qp_init(info, pd->context->device->ibdev_path, iwuqp, resp.wqe_rate_limit_valid); ++ } ++#endif ++ ++ info->sq_size = resp.actual_sq_size; ++ info->rq_size = resp.actual_rq_size; ++ ++ info->qp_caps = resp.qp_caps; ++ info->qp_id = resp.qp_id; ++ iwuqp->zxdh_drv_opt = resp.zxdh_drv_opt; ++ iwuqp->vqp.qp.qp_num = resp.qp_id; ++ iwuqp->qp.is_ext_qp = resp.is_ext_qp; ++ if (iwuqp->qp.is_ext_qp) { ++ iwuqp->qp.ext_sq_base_va = (struct zxdh_qp_sq_quanta *)((__u8 *)info->ext_qp_buff_base + resp.sq_offset); ++ iwuqp->qp.ext_rq_base_va = (struct zxdh_qp_rq_quanta *)((__u8 *)info->ext_qp_buff_base + resp.rq_offset); ++ } ++ ++ iwuqp->send_cq = ++ container_of(attr->send_cq, struct zxdh_ucq, verbs_cq.cq); ++ iwuqp->recv_cq = ++ container_of(attr->recv_cq, struct zxdh_ucq, verbs_cq.cq); ++ iwuqp->send_cq->uqp = iwuqp; ++ iwuqp->recv_cq->uqp = iwuqp; ++ ++ return 0; ++err_qp: ++ ibv_cmd_dereg_mr(&iwuqp->vmr); ++err_dereg_mr: ++ zxdh_free_hw_buf(info->sq, iwuqp->buf_size); ++ return ret; ++} ++#if ZXDH_HAVE_QP_EX ++static void zxdh_wr_local_inv(struct ibv_qp_ex *ibqp, uint32_t invalidate_rkey) ++{ ++ struct zxdh_uqp *qp = container_of(ibqp, struct zxdh_uqp, vqp.qp_ex); ++ struct ibv_send_wr wr = {}; ++ struct ibv_send_wr *bad_wr = NULL; ++ ++ wr.opcode = IBV_WR_LOCAL_INV; ++ wr.invalidate_rkey = invalidate_rkey; ++ ++ zxdh_upost_send(&qp->vqp.qp, &wr, &bad_wr); ++} ++ ++static void zxdh_send_wr_send_inv(struct ibv_qp_ex *ibqp, ++ uint32_t invalidate_rkey) ++{ ++ struct zxdh_uqp *qp = container_of(ibqp, struct zxdh_uqp, vqp.qp_ex); ++ struct ibv_send_wr wr = {}; ++ struct ibv_send_wr *bad_wr = NULL; ++ ++ wr.opcode = IBV_WR_SEND_WITH_INV; ++ wr.invalidate_rkey = invalidate_rkey; ++ ++ zxdh_upost_send(&qp->vqp.qp, &wr, &bad_wr); ++} ++ ++static void zxdh_wr_bind_mw(struct ibv_qp_ex *ibqp, struct ibv_mw *ibmw, ++ uint32_t rkey, const struct ibv_mw_bind_info *info) ++{ ++ struct zxdh_uqp *qp = container_of(ibqp, struct zxdh_uqp, vqp.qp_ex); ++ struct ibv_send_wr wr = {}; ++ struct ibv_send_wr *bad_wr = NULL; ++ ++ if (ibmw->type != IBV_MW_TYPE_2) ++ return; ++ ++ wr.opcode = IBV_WR_BIND_MW; ++ wr.bind_mw.bind_info = *info; ++ wr.bind_mw.mw = ibmw; ++ wr.bind_mw.rkey = rkey; ++ ++ zxdh_upost_send(&qp->vqp.qp, &wr, &bad_wr); ++} ++#endif ++static struct ibv_qp *create_qp(struct ibv_context *ibv_ctx, ++ struct ibv_qp_init_attr_ex *attr_ex) ++{ ++ struct zxdh_qp_init_info info = {}; ++ struct zxdh_dev_attrs *dev_attrs; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_uqp *iwuqp; ++ struct zxdh_usrq *iwusrq; ++ struct ibv_pd *pd = attr_ex->pd; ++ struct ibv_qp_init_attr *attr; ++ __u32 sqdepth, rqdepth; ++ __u8 sqshift, rqshift; ++ int status; ++ __u64 addr_offset = 0; ++ int ret; ++ ++ attr = calloc(1, sizeof(*attr)); ++ if (!attr) ++ return NULL; ++ ++ memcpy(attr, attr_ex, sizeof(*attr)); ++ ++ if (attr->qp_type != IBV_QPT_RC && attr->qp_type != IBV_QPT_UD) { ++ errno = EOPNOTSUPP; ++ free(attr); ++ return NULL; ++ } ++ iwvctx = container_of(ibv_ctx, struct zxdh_uvcontext, ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ ++ if (attr->cap.max_send_sge > dev_attrs->max_hw_wq_frags || ++ attr->cap.max_recv_sge > dev_attrs->max_hw_wq_frags) { ++ errno = EINVAL; ++ free(attr); ++ return NULL; ++ } ++ ++ if (iwvctx->is_qp_ext_mem_support && attr->cap.max_recv_sge > dev_attrs->max_hw_rq_sges) { ++ zxdh_dbg(ZXDH_DBG_SRQ, "zxdh_check_rq_init_attr max_recv_sge failed, when use_blue_flame=true, max_recv_sge is 7.\n"); ++ errno = EINVAL; ++ free(attr); ++ return NULL; ++ } ++ ++ if (attr->cap.max_inline_data > dev_attrs->max_hw_inline) { ++ zxdh_dbg(ZXDH_DBG_QP, "max_inline_data over max_hw_inline\n"); ++ attr->cap.max_inline_data = dev_attrs->max_hw_inline; ++ } ++ ++ zxdh_get_sq_wqe_shift(attr->cap.max_send_sge, attr->cap.max_inline_data, ++ &sqshift); ++ status = zxdh_get_sqdepth(dev_attrs, attr->cap.max_send_wr, sqshift, ++ &sqdepth); ++ if (status) { ++ errno = EINVAL; ++ free(attr); ++ return NULL; ++ } ++ ++ zxdh_get_rq_wqe_shift(attr->cap.max_recv_sge, &rqshift); ++ status = zxdh_get_rqdepth(dev_attrs, attr->cap.max_recv_wr, rqshift, ++ &rqdepth); ++ if (status) { ++ errno = EINVAL; ++ free(attr); ++ return NULL; ++ } ++ ++ iwuqp = memalign(1024, sizeof(*iwuqp)); ++ if (!iwuqp) { ++ free(attr); ++ return NULL; ++ } ++ ++ memset(iwuqp, 0, sizeof(*iwuqp)); ++ iwuqp->wqe_rate_limit_info = NULL; ++#if ZXDH_HAVE_QP_EX ++ if (attr_ex->comp_mask & IBV_QP_INIT_ATTR_SEND_OPS_FLAGS) { ++ if (attr_ex->send_ops_flags & ~IBV_QP_EX_WITH_BIND_MW) { ++ errno = EOPNOTSUPP; ++ free(iwuqp); ++ free(attr); ++ return NULL; ++ } ++ ++ iwuqp->vqp.comp_mask |= VERBS_QP_EX; ++ if (attr_ex->send_ops_flags & IBV_QP_EX_WITH_BIND_MW) ++ iwuqp->vqp.qp_ex.wr_bind_mw = zxdh_wr_bind_mw; ++ ++ if (attr_ex->send_ops_flags & IBV_QP_EX_WITH_SEND_WITH_INV) ++ iwuqp->vqp.qp_ex.wr_send_inv = zxdh_send_wr_send_inv; ++ ++ if (attr_ex->send_ops_flags & IBV_QP_EX_WITH_LOCAL_INV) ++ iwuqp->vqp.qp_ex.wr_local_inv = zxdh_wr_local_inv; ++ } ++#endif ++ if (pthread_spin_init(&iwuqp->lock, PTHREAD_PROCESS_PRIVATE)) ++ goto err_free_qp; ++ if (pthread_spin_init(&iwuqp->quanta_lock, PTHREAD_PROCESS_PRIVATE)) ++ goto err_destroy_quanta_lock; ++ // sqdepth is the number of quanta, not the wqe num ++ // rqdepth is the wqe num, not the number of quanta ++ info.sq_size = sqdepth >> sqshift; ++ info.rq_size = rqdepth; ++ info.rqshift = rqshift; ++ attr->cap.max_send_wr = info.sq_size; ++ attr->cap.max_recv_wr = info.rq_size; ++ ++ info.dev_attrs = dev_attrs; ++ info.max_sq_frag_cnt = attr->cap.max_send_sge; ++ info.max_rq_frag_cnt = attr->cap.max_recv_sge; ++ ++ if (attr->srq != NULL) { ++ iwuqp->is_srq = true; ++ iwusrq = container_of(attr->srq, struct zxdh_usrq, ibv_srq); ++ iwuqp->srq = iwusrq; ++ iwuqp->qp.is_srq = true; ++ } ++ ++ if (iwuqp->is_srq == false) { ++ iwuqp->recv_sges = calloc(attr->cap.max_recv_sge, ++ sizeof(*iwuqp->recv_sges)); ++ if (!iwuqp->recv_sges) ++ goto err_destroy_lock; ++ } ++ ++ addr_offset = ++ (ZXDH_SYS_PAGE_SIZE == ZXDH_HW_64K_PAGE_SIZE) ? ++ (iwvctx->dev_attrs.sq_db_bar_off & (ZXDH_HW_64K_PAGE_SIZE - 1)) : ++ 0; ++ if (iwvctx->is_qp_ext_mem_support) { ++ info.is_qp_ext_mem_support = iwvctx->is_qp_ext_mem_support; ++ info.ext_sq_db = (__u32 *)((__u8 *)iwvctx->ext_sq_db + ZXDH_DB_SQ_OFFSET); ++ info.ext_qp_buff_base = iwvctx->ext_qp_buff_base; ++ } ++ info.wqe_alloc_db = ++ (__u32 *)((__u8 *)iwvctx->sq_db + ZXDH_DB_SQ_OFFSET + addr_offset); ++ info.abi_ver = iwvctx->abi_ver; ++ info.legacy_mode = iwvctx->legacy_mode; ++ info.sq_wrtrk_array = calloc(sqdepth, sizeof(*info.sq_wrtrk_array)); ++ if (!info.sq_wrtrk_array) ++ goto err_free_rsges; ++ ++ if (iwuqp->is_srq == false) { ++ info.rq_wrid_array = ++ calloc(info.rq_size, sizeof(*info.rq_wrid_array)); ++ if (!info.rq_wrid_array) ++ goto err_free_sq_wrtrk; ++ } ++ ++ iwuqp->sq_sig_all = attr->sq_sig_all; ++ iwuqp->qp_type = attr->qp_type; ++ if (attr->qp_type == IBV_QPT_UD) ++ info.type = ZXDH_QP_TYPE_ROCE_UD; ++ else ++ info.type = ZXDH_QP_TYPE_ROCE_RC; ++ status = zxdh_vmapped_qp(iwuqp, pd, attr, sqdepth, rqdepth, &info, ++ iwvctx->legacy_mode); ++ if (status) { ++ errno = status; ++ goto err_free_rq_wrid; ++ } ++ ++ iwuqp->qp.back_qp = iwuqp; ++ iwuqp->qp.lock = &iwuqp->lock; ++ iwuqp->qp.quanta_lock = &iwuqp->quanta_lock; ++ info.max_sq_frag_cnt = attr->cap.max_send_sge; ++ info.max_rq_frag_cnt = attr->cap.max_recv_sge; ++ info.max_inline_data = attr->cap.max_inline_data; ++ if (info.type == ZXDH_QP_TYPE_ROCE_RC) { ++ iwuqp->qp.split_sg_list = ++ calloc(2 * dev_attrs->max_hw_read_sges, ++ sizeof(*iwuqp->qp.split_sg_list)); ++ if (!iwuqp->qp.split_sg_list) ++ goto err_free_vmap_qp; ++ } ++ status = zxdh_qp_init(&iwuqp->qp, &info); ++ if (status) { ++ errno = EINVAL; ++ goto err_free_sg_list; ++ } ++ iwuqp->qp.mtu = mtu_enum_to_int(IBV_MTU_1024); ++ attr->cap.max_send_wr = sqdepth >> sqshift; ++ attr->cap.max_recv_wr = rqdepth; ++ memcpy(attr_ex, attr, sizeof(*attr)); ++ free(attr); ++ return &iwuqp->vqp.qp; ++ ++err_free_sg_list: ++ if (iwuqp->qp.split_sg_list) ++ free(iwuqp->qp.split_sg_list); ++err_free_vmap_qp: ++ zxdh_destroy_vmapped_qp(iwuqp); ++ zxdh_free_hw_buf(info.sq, iwuqp->buf_size); ++err_free_rq_wrid: ++ free(info.rq_wrid_array); ++err_free_sq_wrtrk: ++ free(info.sq_wrtrk_array); ++err_free_rsges: ++ free(iwuqp->recv_sges); ++err_destroy_lock: ++ ret = pthread_spin_destroy(&iwuqp->quanta_lock); ++ if (ret) ++ errno = EINVAL; ++err_destroy_quanta_lock: ++ pthread_spin_destroy(&iwuqp->lock); ++err_free_qp: ++ free(iwuqp); ++ free(attr); ++ ++ return NULL; ++} ++ ++/** ++ * zxdh_ucreate_qp - create qp on user app ++ * @pd: pd for the qp ++ * @attr: attributes of the qp to be created (sizes, sge, cq) ++ */ ++struct ibv_qp *zxdh_ucreate_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr) ++{ ++ struct ibv_qp_init_attr_ex attrx = {}; ++ struct ibv_qp *qp; ++ ++ memcpy(&attrx, attr, sizeof(*attr)); ++ attrx.comp_mask = IBV_QP_INIT_ATTR_PD; ++ attrx.pd = pd; ++ ++ qp = create_qp(pd->context, &attrx); ++ if (qp) ++ memcpy(attr, &attrx, sizeof(*attr)); ++ ++ return qp; ++} ++ ++/** ++ * zxdh_ucreate_qp_ex - create qp_ex on user app ++ * @context: user context of the device ++ * @attr: attributes of the qp_ex to be created ++ */ ++struct ibv_qp *zxdh_ucreate_qp_ex(struct ibv_context *context, ++ struct ibv_qp_init_attr_ex *attr) ++{ ++ return create_qp(context, attr); ++} ++ ++/** ++ * zxdh_uquery_qp - query qp for some attribute ++ * @qp: qp for the attributes query ++ * @attr: to return the attributes ++ * @attr_mask: mask of what is query for ++ * @init_attr: initial attributes during create_qp ++ */ ++int zxdh_uquery_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask, ++ struct ibv_qp_init_attr *init_attr) ++{ ++ struct ibv_query_qp cmd; ++ ++ return ibv_cmd_query_qp(qp, attr, attr_mask, init_attr, &cmd, ++ sizeof(cmd)); ++} ++ ++/** ++ * zxdh_clean_cqes - clean cq entries for qp ++ * @qp: qp for which completions are cleaned ++ * @iwcq: cq to be cleaned ++ */ ++static void zxdh_clean_cqes(struct zxdh_qp *qp, struct zxdh_ucq *iwucq) ++{ ++ struct zxdh_cq *ukcq = &iwucq->cq; ++ int ret; ++ ++ ret = pthread_spin_lock(&iwucq->lock); ++ if (ret) ++ return; ++ ++ zxdh_clean_cq(qp, ukcq); ++ pthread_spin_unlock(&iwucq->lock); ++} ++ ++static void zxdh_init_qp_indices(struct zxdh_qp *qp) ++{ ++ __u32 sq_ring_size; ++ sq_ring_size = ZXDH_RING_SIZE(qp->sq_ring); ++ ZXDH_RING_INIT(qp->sq_ring, sq_ring_size); ++ ZXDH_RING_INIT(qp->initial_ring, sq_ring_size); ++ qp->swqe_polarity = 0; ++ qp->swqe_polarity_deferred = 1; ++ qp->rwqe_polarity = 0; ++ qp->rwqe_signature = 0; ++ ZXDH_RING_INIT(qp->rq_ring, qp->rq_size); ++ /* 新增：初始化last_sqe_idx和last_rqe_idx */ ++ qp->last_sqe_idx = 0; ++ qp->last_rqe_idx = 0; ++} ++ ++/** ++ * zxdh_umodify_qp - send qp modify to driver ++ * @qp: qp to modify ++ * @attr: attribute to modify ++ * @attr_mask: mask of the attribute ++ */ ++int zxdh_umodify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask) ++{ ++ struct zxdh_uqp *iwuqp; ++ struct zxdh_umodify_qp_resp resp = {}; ++ struct ibv_modify_qp cmd = {}; ++ struct zxdh_umodify_qp cmd_ex = {}; ++ int ret; ++ __u16 mtu = 0; ++ ++ iwuqp = container_of(qp, struct zxdh_uqp, vqp.qp); ++ if (attr_mask & IBV_QP_CAP) ++ return -EINVAL; ++ if (attr_mask & IBV_QP_PATH_MTU) { ++ if (attr->path_mtu < IBV_MTU_256 || attr->path_mtu > IBV_MTU_4096) { ++ return -EINVAL; ++ } ++ } ++ ++ if (attr_mask & IBV_QP_STATE || attr_mask & IBV_QP_RATE_LIMIT) { ++ ret = ibv_cmd_modify_qp_ex(qp, attr, attr_mask, &cmd_ex.ibv_cmd, ++ sizeof(cmd_ex), &resp.ibv_resp, ++ sizeof(resp)); ++ } else { ++ ret = ibv_cmd_modify_qp(qp, attr, attr_mask, &cmd, sizeof(cmd)); ++ } ++ ++ if (!ret && (attr_mask & IBV_QP_STATE) && ++ attr->qp_state == IBV_QPS_RESET) { ++ if (iwuqp->send_cq) ++ zxdh_clean_cqes(&iwuqp->qp, iwuqp->send_cq); ++ ++ if (iwuqp->recv_cq && iwuqp->recv_cq != iwuqp->send_cq) ++ zxdh_clean_cqes(&iwuqp->qp, iwuqp->recv_cq); ++ memset(iwuqp->qp.sq_base, 0, iwuqp->buf_size); ++ zxdh_init_qp_indices(&iwuqp->qp); ++ } ++ ++ if (!ret && (attr_mask & IBV_QP_PATH_MTU) && ++ qp->qp_type == IBV_QPT_RC) { ++ mtu = mtu_enum_to_int(attr->path_mtu); ++ if (mtu == 0) ++ return -EINVAL; ++ iwuqp->qp.mtu = mtu; ++ } ++ if (!ret && (attr_mask & IBV_QP_SQ_PSN) && qp->qp_type == IBV_QPT_RC) { ++ iwuqp->qp.next_psn = attr->sq_psn; ++ iwuqp->qp.cqe_last_ack_qsn = attr->sq_psn - 1; ++ iwuqp->qp.qp_last_ack_qsn = attr->sq_psn - 1; ++ iwuqp->qp.cqe_retry_cnt = 0; ++ iwuqp->qp.qp_reset_cnt = 0; ++ } ++ return ret; ++} ++ ++static void zxdh_issue_flush(struct ibv_qp *qp, bool sq_flush, bool rq_flush) ++{ ++ struct ib_uverbs_ex_modify_qp_resp resp = {}; ++ struct zxdh_umodify_qp cmd_ex = {}; ++ struct ibv_qp_attr attr = {}; ++ ++ attr.qp_state = IBV_QPS_ERR; ++ cmd_ex.sq_flush = sq_flush; ++ cmd_ex.rq_flush = rq_flush; ++ ++ ibv_cmd_modify_qp_ex(qp, &attr, IBV_QP_STATE, &cmd_ex.ibv_cmd, ++ sizeof(cmd_ex), &resp, sizeof(resp)); ++} ++ ++/** ++ * zxdh_udestroy_qp - destroy qp ++ * @qp: qp to destroy ++ */ ++int zxdh_udestroy_qp(struct ibv_qp *qp) ++{ ++ struct zxdh_uqp *iwuqp; ++ int ret; ++ ++ iwuqp = container_of(qp, struct zxdh_uqp, vqp.qp); ++ ret = pthread_spin_destroy(&iwuqp->quanta_lock); ++ if (ret) ++ goto err; ++ iwuqp->qp.quanta_lock = NULL; ++ ret = pthread_spin_destroy(&iwuqp->lock); ++ if (ret) ++ goto err; ++ iwuqp->qp.lock = NULL; ++ ++ iwuqp->qp.destroy_pending = true; ++ ++ ret = zxdh_destroy_vmapped_qp(iwuqp); ++ if (ret) ++ goto err; ++#if defined(__x86_64__) || defined (__i386__) ++ if (iwuqp->wqe_rate_limit_info) { ++ rdma_wqe_rate_limit_qp_exit(iwuqp); ++ } ++#endif ++ /* Clean any pending completions from the cq(s) */ ++ if (iwuqp->send_cq) ++ zxdh_clean_cqes(&iwuqp->qp, iwuqp->send_cq); ++ ++ if (iwuqp->recv_cq && iwuqp->recv_cq != iwuqp->send_cq) ++ zxdh_clean_cqes(&iwuqp->qp, iwuqp->recv_cq); ++ ++ if (iwuqp->qp.sq_wrtrk_array) ++ free(iwuqp->qp.sq_wrtrk_array); ++ if (iwuqp->qp.rq_wrid_array) ++ free(iwuqp->qp.rq_wrid_array); ++ if (iwuqp->qp.split_sg_list) ++ free(iwuqp->qp.split_sg_list); ++ ++ if (iwuqp->ext_buf_mode == 0) ++ zxdh_free_hw_buf(iwuqp->qp.sq_base, iwuqp->buf_size); ++ ++ if (iwuqp->recv_sges) ++ free(iwuqp->recv_sges); ++ ++ free(iwuqp); ++ return 0; ++ ++err: ++ return ret; ++} ++ ++/** ++ * zxdh_copy_sg_list - copy sg list for qp ++ * @sg_list: copied into sg_list ++ * @sgl: copy from sgl ++ * @num_sges: count of sg entries ++ * @max_sges: count of max supported sg entries ++ */ ++static void zxdh_copy_sg_list(struct zxdh_sge *sg_list, struct ibv_sge *sgl, ++ int num_sges) ++{ ++ int i; ++ ++ for (i = 0; i < num_sges; i++) { ++ sg_list[i].tag_off = sgl[i].addr; ++ sg_list[i].len = sgl[i].length; ++ sg_list[i].stag = sgl[i].lkey; ++ } ++} ++ ++/** ++ * calc_type2_mw_stag - calculate type 2 MW stag ++ * @rkey: desired rkey of the MW ++ * @mw_rkey: type2 memory window rkey ++ * ++ * compute type2 memory window stag by taking lower 8 bits ++ * of the desired rkey and leaving 24 bits if mw->rkey unchanged ++ */ ++static inline __u32 calc_type2_mw_stag(__u32 rkey, __u32 mw_rkey) ++{ ++ const __u32 mask = 0xff; ++ ++ return (rkey & mask) | (mw_rkey & ~mask); ++} ++ ++/** ++ * zxdh_post_send - post send wr for user application ++ * @ib_qp: qp to post wr ++ * @ib_wr: work request ptr ++ * @bad_wr: return of bad wr if err ++ */ ++int zxdh_upost_send(struct ibv_qp *ib_qp, struct ibv_send_wr *ib_wr, ++ struct ibv_send_wr **bad_wr) ++{ ++ struct zxdh_post_sq_info info; ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ enum zxdh_status_code ret = 0; ++ struct zxdh_uqp *iwuqp; ++ bool reflush = false; ++ int err = 0; ++ errno = 0; ++ struct verbs_mr *vmr = NULL; ++ struct zxdh_umr *umr = NULL; ++ __u64 mr_va = 0, mw_va = 0, value_dffer = 0, mw_pa_pble_index = 0; ++ __u16 mr_offset = 0; ++ ++ iwuqp = container_of(ib_qp, struct zxdh_uqp, vqp.qp); ++ iwvctx = container_of(ib_qp->context, struct zxdh_uvcontext, ++ ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ ++ err = pthread_spin_lock(&iwuqp->lock); ++ if (err) ++ return err; ++ ++ if (!ZXDH_RING_MORE_WORK(iwuqp->qp.sq_ring) && ++ ib_qp->state == IBV_QPS_ERR) ++ reflush = true; ++ ++ while (ib_wr) { ++ memset(&info, 0, sizeof(info)); ++ info.wr_id = (__u64)(ib_wr->wr_id); ++ if ((ib_wr->send_flags & IBV_SEND_SIGNALED) || ++ iwuqp->sq_sig_all) ++ info.signaled = true; ++ if (ib_wr->send_flags & IBV_SEND_FENCE) ++ info.read_fence = true; ++ ++ switch (ib_wr->opcode) { ++ case IBV_WR_SEND_WITH_IMM: ++ if (iwuqp->qp.qp_caps & ZXDH_SEND_WITH_IMM) { ++ info.imm_data_valid = true; ++ info.imm_data = ntohl(ib_wr->imm_data); ++ } else { ++ zxdh_status_to_errno_log(ZXDH_NOT_SUPPORTED, ++ "qp not supported send with imm, qp_id=%d", ++ iwuqp->qp.qp_id); ++ break; ++ } ++ SWITCH_FALLTHROUGH; ++ case IBV_WR_SEND: ++ case IBV_WR_SEND_WITH_INV: ++ if (ib_wr->send_flags & IBV_SEND_SOLICITED) ++ info.solicited = 1; ++ ++ if (ib_wr->opcode == IBV_WR_SEND) { ++ if (ib_qp->qp_type == IBV_QPT_UD) ++ info.op_type = ZXDH_OP_TYPE_UD_SEND; ++ else ++ info.op_type = ZXDH_OP_TYPE_SEND; ++ } else if (ib_wr->opcode == IBV_WR_SEND_WITH_IMM) { ++ if (ib_qp->qp_type == IBV_QPT_UD) ++ info.op_type = ++ ZXDH_OP_TYPE_UD_SEND_WITH_IMM; ++ else ++ info.op_type = ++ ZXDH_OP_TYPE_SEND_WITH_IMM; ++ } else { ++ info.op_type = ZXDH_OP_TYPE_SEND_INV; ++ info.stag_to_inv = ib_wr->invalidate_rkey; ++ } ++ ++ if ((ib_wr->send_flags & IBV_SEND_INLINE) && ++ (ib_wr->num_sge != 0)) { ++ ret = zxdh_get_inline_data( ++ iwuqp->inline_data, ib_wr, ++ &info.op.inline_rdma_send.len); ++ if (ret) { ++ zxdh_status_to_errno_log(ret,"qp get inline data failed, qp_id=%d", iwuqp->qp.qp_id); ++ break; ++ } ++ info.op.inline_rdma_send.data = ++ iwuqp->inline_data; ++ if (ib_qp->qp_type == IBV_QPT_UD) { ++ struct zxdh_uah *ah = ++ container_of(ib_wr->wr.ud.ah, ++ struct zxdh_uah, ++ ibv_ah); ++ info.op.inline_rdma_send.ah_id = ++ ah->ah_id; ++ info.op.inline_rdma_send.qkey = ++ ib_wr->wr.ud.remote_qkey; ++ info.op.inline_rdma_send.dest_qp = ++ ib_wr->wr.ud.remote_qpn; ++ ret = zxdh_ud_inline_send(&iwuqp->qp, ++ &info, false); ++ } else { ++ ret = zxdh_rc_inline_send(&iwuqp->qp, ++ &info, false); ++ } ++ } else { ++ info.op.send.num_sges = ib_wr->num_sge; ++ info.op.send.sg_list = ++ (struct zxdh_sge *)ib_wr->sg_list; ++ if (ib_qp->qp_type == IBV_QPT_UD) { ++ struct zxdh_uah *ah = ++ container_of(ib_wr->wr.ud.ah, ++ struct zxdh_uah, ++ ibv_ah); ++ ++ info.op.inline_rdma_send.ah_id = ++ ah->ah_id; ++ info.op.inline_rdma_send.qkey = ++ ib_wr->wr.ud.remote_qkey; ++ info.op.inline_rdma_send.dest_qp = ++ ib_wr->wr.ud.remote_qpn; ++ ret = zxdh_ud_send(&iwuqp->qp, &info, ++ false); ++ } else { ++ ret = zxdh_rc_send(&iwuqp->qp, &info, ++ false); ++ } ++ } ++ if (ret) ++ zxdh_status_to_errno_log(ret,"qp send data failed, qp_id=%d", iwuqp->qp.qp_id); ++ break; ++ case IBV_WR_RDMA_WRITE_WITH_IMM: ++ if (iwuqp->qp.qp_caps & ZXDH_WRITE_WITH_IMM) { ++ info.imm_data_valid = true; ++ info.imm_data = ntohl(ib_wr->imm_data); ++ } else { ++ zxdh_status_to_errno_log(ZXDH_NOT_SUPPORTED, ++ "qp not supported write with imm, qp_id=%d", ++ iwuqp->qp.qp_id); ++ break; ++ } ++ SWITCH_FALLTHROUGH; ++ case IBV_WR_RDMA_WRITE: ++ if (ib_wr->send_flags & IBV_SEND_SOLICITED) ++ info.solicited = 1; ++ ++ if (ib_wr->opcode == IBV_WR_RDMA_WRITE) ++ info.op_type = ZXDH_OP_TYPE_WRITE; ++ else ++ info.op_type = ZXDH_OP_TYPE_WRITE_WITH_IMM; ++ ++ if ((ib_wr->send_flags & IBV_SEND_INLINE) && ++ (ib_wr->num_sge != 0)) { ++ ret = zxdh_get_inline_data( ++ iwuqp->inline_data, ib_wr, ++ &info.op.inline_rdma_write.len); ++ if (ret) { ++ zxdh_status_to_errno_log(ret,"qp get inline data failed, qp_id=%d", iwuqp->qp.qp_id); ++ break; ++ } ++ info.op.inline_rdma_write.data = ++ iwuqp->inline_data; ++ info.op.inline_rdma_write.rem_addr.tag_off = ++ ib_wr->wr.rdma.remote_addr; ++ info.op.inline_rdma_write.rem_addr.stag = ++ ib_wr->wr.rdma.rkey; ++ ret = zxdh_inline_rdma_write(&iwuqp->qp, &info, ++ false); ++ } else { ++ info.op.rdma_write.lo_sg_list = ++ (void *)ib_wr->sg_list; ++ info.op.rdma_write.num_lo_sges = ib_wr->num_sge; ++ info.op.rdma_write.rem_addr.tag_off = ++ ib_wr->wr.rdma.remote_addr; ++ info.op.rdma_write.rem_addr.stag = ++ ib_wr->wr.rdma.rkey; ++ ret = zxdh_rdma_write(&iwuqp->qp, &info, false); ++ } ++ if (ret) ++ zxdh_status_to_errno_log(ret,"qp write data failed, qp_id=%d", iwuqp->qp.qp_id); ++ break; ++ case IBV_WR_RDMA_READ: ++ if (ib_wr->num_sge > dev_attrs->max_hw_read_sges) { ++ zxdh_status_to_errno_log(ZXDH_ERR_INVALID_FRAG_COUNT,"qp invalid sge number, qp_id=%d", iwuqp->qp.qp_id); ++ break; ++ } ++ info.op_type = ZXDH_OP_TYPE_READ; ++ info.op.rdma_read.rem_addr.tag_off = ++ ib_wr->wr.rdma.remote_addr; ++ info.op.rdma_read.rem_addr.stag = ib_wr->wr.rdma.rkey; ++ ++ info.op.rdma_read.lo_sg_list = (void *)ib_wr->sg_list; ++ info.op.rdma_read.num_lo_sges = ib_wr->num_sge; ++#if defined(__x86_64__) || defined (__i386__) ++ if (iwuqp->wqe_rate_limit_info && iwuqp->wqe_rate_limit_info->wqe_rate_limit) { ++ rdma_qp_delay_wqe(&info, iwuqp); ++ } ++#endif ++ ret = zxdh_rdma_read(&iwuqp->qp, &info, false, false); ++ if (ret) ++ zxdh_status_to_errno_log(ret,"qp read date failed, qp_id=%d", iwuqp->qp.qp_id); ++ break; ++ case IBV_WR_BIND_MW: ++ vmr = verbs_get_mr(ib_wr->bind_mw.bind_info.mr); ++ umr = container_of(vmr, struct zxdh_umr, vmr); ++ mr_va = (uintptr_t)ib_wr->bind_mw.bind_info.mr->addr; ++ mw_va = ib_wr->bind_mw.bind_info.addr; ++ mr_offset = 0; ++ value_dffer = 0; ++ mw_pa_pble_index = 0; ++ ++ if (ib_qp->qp_type != IBV_QPT_RC) { ++ zxdh_status_to_errno_log(ZXDH_NOT_SUPPORTED,"qp type not supported bind mw, qp_id=%d", iwuqp->qp.qp_id); ++ break; ++ } ++ info.op_type = ZXDH_OP_TYPE_BIND_MW; ++ info.op.bind_window.mr_stag = ++ ib_wr->bind_mw.bind_info.mr->rkey; ++ ++ if (ib_wr->bind_mw.mw->type == IBV_MW_TYPE_1) { ++ info.op.bind_window.mem_window_type_1 = true; ++ info.op.bind_window.mw_stag = ++ ib_wr->bind_mw.rkey; ++ } else { ++ info.op.bind_window.mem_window_type_1 = false; ++ info.op.bind_window.mw_stag = ++ calc_type2_mw_stag( ++ ib_wr->bind_mw.rkey, ++ ib_wr->bind_mw.mw->rkey); ++ ib_wr->bind_mw.mw->rkey = ++ info.op.bind_window.mw_stag; ++ } ++ ++ if (ib_wr->bind_mw.bind_info.mw_access_flags & ++ IBV_ACCESS_ZERO_BASED) { ++ info.op.bind_window.addressing_type = ++ ZXDH_ADDR_TYPE_ZERO_BASED; ++ if (ib_wr->bind_mw.mw->type == IBV_MW_TYPE_1) { ++ zxdh_status_to_errno_log(ZXDH_ERR_BIND_MW_TYPE,"bind mw type not supported bind mw, qp_id=%d", iwuqp->qp.qp_id); ++ break; ++ } ++ ++ info.op.bind_window.addressing_type = ++ ZXDH_ADDR_TYPE_ZERO_BASED; ++ info.op.bind_window.host_page_size = ++ umr->host_page_size; ++ if (umr->host_page_size == ZXDH_PAGE_SIZE_4K) { ++ mr_offset = mr_va & 0x0fff; ++ value_dffer = mw_va - mr_va; ++ if (umr->leaf_pbl_size == 3) { ++ mw_pa_pble_index = ++ (mr_offset + ++ value_dffer) / ++ (4096 * 512); ++ info.op.bind_window ++ .mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ mw_pa_pble_index; ++ mw_pa_pble_index = ++ ((mr_offset + ++ value_dffer) / ++ 4096) % ++ 512; ++ ++ info.op.bind_window ++ .root_leaf_offset = ++ (__u16)mw_pa_pble_index; ++ info.op.bind_window.va = ++ (void *)(uintptr_t)(mw_va & ++ 0x0fff); ++ info.op.bind_window ++ .leaf_pbl_size = 3; ++ ++ } else if (umr->leaf_pbl_size == 1) { ++ mw_pa_pble_index = ++ (mr_offset + ++ value_dffer) / ++ 4096; ++ info.op.bind_window ++ .mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ mw_pa_pble_index; ++ info.op.bind_window ++ .leaf_pbl_size = 1; ++ info.op.bind_window.va = ++ (void *)(uintptr_t)(mw_va & ++ 0x0fff); ++ info.op.bind_window ++ .root_leaf_offset = 0; ++ } else { ++ mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ mr_offset + value_dffer; ++ info.op.bind_window.va = ++ (void *)(uintptr_t)(mw_va & ++ 0x0fff); ++ info.op.bind_window ++ .mw_pa_pble_index = ++ mw_pa_pble_index; ++ info.op.bind_window ++ .leaf_pbl_size = 0; ++ info.op.bind_window ++ .root_leaf_offset = 0; ++ } ++ } else if (umr->host_page_size == ZXDH_PAGE_SIZE_64K) { ++ mr_offset = mr_va & 0xFFFF; ++ value_dffer = mw_va - mr_va; ++ if (umr->leaf_pbl_size == 3) { ++ mw_pa_pble_index = ++ (mr_offset + ++ value_dffer) / ++ ((4096 * 16) * 512); ++ info.op.bind_window ++ .mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ mw_pa_pble_index; ++ mw_pa_pble_index = ++ ((mr_offset + ++ value_dffer) / ++ (4096 * 16)) % 512; ++ info.op.bind_window ++ .root_leaf_offset = ++ (__u16)mw_pa_pble_index; ++ info.op.bind_window.va = ++ (void *)(uintptr_t)(mw_va & ++ 0xFFFF); ++ info.op.bind_window ++ .leaf_pbl_size = 3; ++ } else if (umr->leaf_pbl_size == 1) { ++ mw_pa_pble_index = ++ (mr_offset + ++ value_dffer) / ++ (4096 * 16); ++ info.op.bind_window ++ .mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ mw_pa_pble_index; ++ info.op.bind_window ++ .leaf_pbl_size = 1; ++ info.op.bind_window.va = ++ (void *)(uintptr_t)(mw_va & ++ 0xFFFF); ++ } else { ++ mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ mr_offset + value_dffer; ++ info.op.bind_window.va = ++ (void *)(uintptr_t)(mw_va & ++ 0xFFFF); ++ info.op.bind_window ++ .mw_pa_pble_index = ++ mw_pa_pble_index; ++ info.op.bind_window ++ .leaf_pbl_size = 0; ++ info.op.bind_window ++ .root_leaf_offset = 0; ++ } ++ } else if (umr->host_page_size == ++ ZXDH_PAGE_SIZE_2M) { ++ mr_offset = mr_va & 0x1FFFFF; ++ value_dffer = mw_va - mr_va; ++ if (umr->leaf_pbl_size == 3) { ++ mw_pa_pble_index = ++ (mr_offset + ++ value_dffer) / ++ ((4096 * 512) * 512); ++ info.op.bind_window ++ .mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ mw_pa_pble_index; ++ mw_pa_pble_index = ++ ((mr_offset + ++ value_dffer) / ++ (4096 * 512)) % ++ 512; ++ ++ info.op.bind_window ++ .root_leaf_offset = ++ (__u16)mw_pa_pble_index; ++ info.op.bind_window.va = ++ (void *)(uintptr_t)(mw_va & ++ 0x1FFFFF); ++ info.op.bind_window ++ .leaf_pbl_size = 3; ++ ++ } else if (umr->leaf_pbl_size == 1) { ++ mw_pa_pble_index = ++ (mr_offset + ++ value_dffer) / ++ (4096 * 512); ++ info.op.bind_window ++ .mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ mw_pa_pble_index; ++ info.op.bind_window ++ .leaf_pbl_size = 1; ++ info.op.bind_window.va = ++ (void *)(uintptr_t)(mw_va & ++ 0x1FFFFF); ++ info.op.bind_window ++ .root_leaf_offset = 0; ++ } else { ++ mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ mr_offset + value_dffer; ++ info.op.bind_window.va = ++ (void *)(uintptr_t)(mw_va & ++ 0x1FFFFF); ++ info.op.bind_window ++ .mw_pa_pble_index = ++ mw_pa_pble_index; ++ info.op.bind_window ++ .leaf_pbl_size = 0; ++ info.op.bind_window ++ .root_leaf_offset = 0; ++ } ++ } else if (umr->host_page_size == ++ ZXDH_PAGE_SIZE_1G) { ++ mr_offset = mr_va & 0x3FFFFFFF; ++ value_dffer = mw_va - mr_va; ++ if (umr->leaf_pbl_size == 1) { ++ mw_pa_pble_index = ++ (mr_offset + ++ value_dffer) / ++ (1024 * 1024 * 1024); ++ info.op.bind_window ++ .mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ mw_pa_pble_index; ++ info.op.bind_window ++ .leaf_pbl_size = 1; ++ info.op.bind_window.va = ++ (void *)(uintptr_t)(mw_va & ++ 0x3FFFFFFF); ++ info.op.bind_window ++ .root_leaf_offset = 0; ++ } else if (umr->leaf_pbl_size == 0) { ++ mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ mr_offset + value_dffer; ++ info.op.bind_window.va = ++ (void *)(uintptr_t)(mw_va & ++ 0x3FFFFFFF); ++ info.op.bind_window ++ .mw_pa_pble_index = ++ mw_pa_pble_index; ++ info.op.bind_window ++ .leaf_pbl_size = 0; ++ info.op.bind_window ++ .root_leaf_offset = 0; ++ } ++ } ++ ++ } else { ++ info.op.bind_window.addressing_type = ++ ZXDH_ADDR_TYPE_VA_BASED; ++ info.op.bind_window.va = ++ (void *)(uintptr_t) ++ ib_wr->bind_mw.bind_info.addr; ++ info.op.bind_window.host_page_size = ++ umr->host_page_size; ++ ++ if (umr->host_page_size == ZXDH_PAGE_SIZE_4K) { ++ mr_offset = mr_va & 0x0fff; ++ value_dffer = mw_va - mr_va; ++ if (umr->leaf_pbl_size == 3) { ++ mw_pa_pble_index = ++ (mr_offset + ++ value_dffer) / ++ (4096 * 512); ++ info.op.bind_window ++ .mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ mw_pa_pble_index; ++ mw_pa_pble_index = ++ ((mr_offset + ++ value_dffer) / ++ 4096) % ++ 512; ++ info.op.bind_window ++ .root_leaf_offset = ++ (__u16)mw_pa_pble_index; ++ info.op.bind_window ++ .leaf_pbl_size = 3; ++ } else if (umr->leaf_pbl_size == 1) { ++ info.op.bind_window ++ .mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ ((mr_offset + ++ value_dffer) / ++ 4096); ++ info.op.bind_window ++ .leaf_pbl_size = 1; ++ info.op.bind_window ++ .root_leaf_offset = 0; ++ } else { ++ info.op.bind_window ++ .leaf_pbl_size = 0; ++ info.op.bind_window ++ .mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ (mr_va & 0x0fff) + ++ (mw_va - mr_va); ++ info.op.bind_window ++ .root_leaf_offset = 0; ++ } ++ } else if (umr->host_page_size == ZXDH_PAGE_SIZE_64K) { ++ mr_offset = mr_va & 0xFFFF; ++ value_dffer = mw_va - mr_va; ++ if (umr->leaf_pbl_size == 3) { ++ mw_pa_pble_index = ++ (mr_offset + ++ value_dffer) / ++ (4096 * 16); ++ info.op.bind_window ++ .mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ mw_pa_pble_index; ++ mw_pa_pble_index = ++ ((mr_offset + ++ value_dffer) / ++ (4096 * 16)) % ++ 512; ++ info.op.bind_window ++ .root_leaf_offset = ++ (__u16)mw_pa_pble_index; ++ info.op.bind_window ++ .leaf_pbl_size = 3; ++ } else if (umr->leaf_pbl_size == 1) { ++ info.op.bind_window ++ .mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ ((mr_offset + ++ value_dffer) / ++ (4096 * 16)); ++ info.op.bind_window ++ .leaf_pbl_size = 1; ++ info.op.bind_window ++ .root_leaf_offset = 0; ++ } else { ++ info.op.bind_window ++ .leaf_pbl_size = 0; ++ info.op.bind_window ++ .mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ (mr_va & 0xFFFF) + ++ (mw_va - mr_va); ++ info.op.bind_window ++ .root_leaf_offset = 0; ++ } ++ } else if (umr->host_page_size == ++ ZXDH_PAGE_SIZE_2M) { ++ mr_offset = mr_va & 0x1FFFFF; ++ value_dffer = mw_va - mr_va; ++ if (umr->leaf_pbl_size == 3) { ++ mw_pa_pble_index = ++ (mr_offset + ++ value_dffer) / ++ ((4096 * 512) * 512); ++ info.op.bind_window ++ .mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ mw_pa_pble_index; ++ mw_pa_pble_index = ++ ((mr_offset + ++ value_dffer) / ++ (4096 * 512)) % ++ 512; ++ info.op.bind_window ++ .root_leaf_offset = ++ (__u16)mw_pa_pble_index; ++ info.op.bind_window ++ .leaf_pbl_size = 3; ++ } else if (umr->leaf_pbl_size == 1) { ++ info.op.bind_window ++ .mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ ((mr_offset + ++ value_dffer) / ++ (4096 * 512)); ++ info.op.bind_window ++ .leaf_pbl_size = 1; ++ info.op.bind_window ++ .root_leaf_offset = 0; ++ } else { ++ info.op.bind_window ++ .leaf_pbl_size = 0; ++ info.op.bind_window ++ .mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ (mr_va & 0x1FFFFF) + ++ (mw_va - mr_va); ++ info.op.bind_window ++ .root_leaf_offset = 0; ++ } ++ } else if (umr->host_page_size == ++ ZXDH_PAGE_SIZE_1G) { ++ mr_offset = mr_va & 0x3FFFFFFF; ++ value_dffer = mw_va - mr_va; ++ if (umr->leaf_pbl_size == 1) { ++ info.op.bind_window ++ .mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ ((mr_offset + ++ value_dffer) / ++ (1024 * 1024 * 1024)); ++ info.op.bind_window ++ .leaf_pbl_size = 1; ++ info.op.bind_window ++ .root_leaf_offset = 0; ++ } else if (umr->leaf_pbl_size == 0) { ++ info.op.bind_window ++ .leaf_pbl_size = 0; ++ info.op.bind_window ++ .mw_pa_pble_index = ++ umr->mr_pa_pble_index + ++ (mr_va & 0x3FFFFFFF) + ++ (mw_va - mr_va); ++ info.op.bind_window ++ .root_leaf_offset = 0; ++ } ++ } ++ } ++ ++ info.op.bind_window.bind_len = ++ ib_wr->bind_mw.bind_info.length; ++ info.op.bind_window.ena_reads = ++ (ib_wr->bind_mw.bind_info.mw_access_flags & ++ IBV_ACCESS_REMOTE_READ) ? ++ 1 : ++ 0; ++ info.op.bind_window.ena_writes = ++ (ib_wr->bind_mw.bind_info.mw_access_flags & ++ IBV_ACCESS_REMOTE_WRITE) ? ++ 1 : ++ 0; ++ ++ ret = zxdh_mw_bind(&iwuqp->qp, &info, false); ++ if (ret) ++ zxdh_status_to_errno_log(ret,"bind mw failed, qp_id=%d", iwuqp->qp.qp_id); ++ ++ break; ++ case IBV_WR_LOCAL_INV: ++ info.op_type = ZXDH_OP_TYPE_LOCAL_INV; ++ info.op.inv_local_stag.target_stag = ++ ib_wr->invalidate_rkey; ++ ret = zxdh_stag_local_invalidate(&iwuqp->qp, &info, ++ true); ++ if (ret) ++ zxdh_status_to_errno_log(ret,"qp local invalidate failed, qp_id=%d", iwuqp->qp.qp_id); ++ break; ++ default: ++ /* error */ ++ zxdh_status_to_errno_log(ZXDH_ERR_INVALID_WR_OP_TYPE,"invalid wr type, qp_id=%d", iwuqp->qp.qp_id); ++ break; ++ } ++ if (errno != 0) ++ break; ++ ++ ib_wr = ib_wr->next; ++ } ++ ++ if (errno != 0) ++ *bad_wr = ib_wr; ++ err = errno; ++ zxdh_qp_post_wr(&iwuqp->qp); ++ if (reflush) ++ zxdh_issue_flush(ib_qp, 1, 0); ++ ++ pthread_spin_unlock(&iwuqp->lock); ++ ++ return err; ++} ++ ++/** ++ * zxdh_post_recv - post receive wr for user application ++ * @ib_wr: work request for receive ++ * @bad_wr: bad wr caused an error ++ */ ++int zxdh_upost_recv(struct ibv_qp *ib_qp, struct ibv_recv_wr *ib_wr, ++ struct ibv_recv_wr **bad_wr) ++{ ++ struct zxdh_post_rq_info post_recv = {}; ++ enum zxdh_status_code ret = 0; ++ struct zxdh_sge *sg_list; ++ struct zxdh_uqp *iwuqp; ++ bool reflush = false; ++ int err = 0; ++ errno = 0; ++ ++ iwuqp = container_of(ib_qp, struct zxdh_uqp, vqp.qp); ++ sg_list = iwuqp->recv_sges; ++ ++ if (unlikely(ib_qp->state == IBV_QPS_RESET || ib_qp->srq)) { ++ *bad_wr = ib_wr; ++ zxdh_status_to_errno_log(ZXDH_ERR_BAD_QP,"post recv at reset or using srq, qp_id=%d", iwuqp->qp.qp_id); ++ return errno; ++ } ++ ++ err = pthread_spin_lock(&iwuqp->lock); ++ if (err) ++ return err; ++ ++ if (unlikely(!ZXDH_RING_MORE_WORK(iwuqp->qp.rq_ring)) && ++ ib_qp->state == IBV_QPS_ERR) ++ reflush = true; ++ ++ while (ib_wr) { ++ if (unlikely(ib_wr->num_sge > iwuqp->qp.max_rq_frag_cnt)) { ++ *bad_wr = ib_wr; ++ zxdh_status_to_errno_log(ZXDH_ERR_INVALID_FRAG_COUNT,"qp invalid sge number, qp_id=%d", iwuqp->qp.qp_id); ++ goto error; ++ } ++ post_recv.num_sges = ib_wr->num_sge; ++ post_recv.wr_id = ib_wr->wr_id; ++ zxdh_copy_sg_list(sg_list, ib_wr->sg_list, ib_wr->num_sge); ++ post_recv.sg_list = sg_list; ++ ret = zxdh_post_receive(&iwuqp->qp, &post_recv); ++ if (unlikely(ret)) { ++ zxdh_status_to_errno_log(ret,"qp post recv failed, qp_id=%d", iwuqp->qp.qp_id); ++ *bad_wr = ib_wr; ++ goto error; ++ } ++ ++ if (reflush) ++ zxdh_issue_flush(ib_qp, 0, 1); ++ ++ ib_wr = ib_wr->next; ++ } ++error: ++ err = errno; ++ zxdh_qp_set_shadow_area(&iwuqp->qp); ++ pthread_spin_unlock(&iwuqp->lock); ++ ++ return err; ++} ++ ++/** ++ * zxdh_ucreate_ah - create address handle associated with a pd ++ * @ibpd: pd for the address handle ++ * @attr: attributes of address handle ++ */ ++struct ibv_ah *zxdh_ucreate_ah(struct ibv_pd *ibpd, struct ibv_ah_attr *attr) ++{ ++ struct zxdh_uah *ah; ++ union ibv_gid sgid; ++ struct zxdh_ucreate_ah_resp resp; ++ int err; ++ ++ memset(&resp, 0, sizeof(resp)); ++ err = ibv_query_gid(ibpd->context, attr->port_num, attr->grh.sgid_index, ++ &sgid); ++ if (err) { ++ errno = err; ++ return NULL; ++ } ++ ++ ah = calloc(1, sizeof(*ah)); ++ if (!ah) ++ return NULL; ++ ++ err = ibv_cmd_create_ah(ibpd, &ah->ibv_ah, attr, &resp.ibv_resp, ++ sizeof(resp)); ++ if (err) { ++ free(ah); ++ errno = err; ++ return NULL; ++ } ++ ++ ah->ah_id = resp.ah_id; ++ ++ return &ah->ibv_ah; ++} ++ ++/** ++ * zxdh_udestroy_ah - destroy the address handle ++ * @ibah: address handle ++ */ ++int zxdh_udestroy_ah(struct ibv_ah *ibah) ++{ ++ struct zxdh_uah *ah; ++ int ret; ++ ++ ah = container_of(ibah, struct zxdh_uah, ibv_ah); ++ ++ ret = ibv_cmd_destroy_ah(ibah); ++ if (ret) ++ return ret; ++ ++ free(ah); ++ ++ return 0; ++} ++ ++/** ++ * zxdh_uattach_mcast - Attach qp to multicast group implemented ++ * @qp: The queue pair ++ * @gid:The Global ID for multicast group ++ * @lid: The Local ID ++ */ ++int zxdh_uattach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, ++ uint16_t lid) ++{ ++ return ibv_cmd_attach_mcast(qp, gid, lid); ++} ++ ++/** ++ * zxdh_udetach_mcast - Detach qp from multicast group ++ * @qp: The queue pair ++ * @gid:The Global ID for multicast group ++ * @lid: The Local ID ++ */ ++int zxdh_udetach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, ++ uint16_t lid) ++{ ++ return ibv_cmd_detach_mcast(qp, gid, lid); ++} ++ ++/** ++ * zxdh_uresize_cq - resizes a cq ++ * @cq: cq to resize ++ * @cqe: the number of cqes of the new cq ++ */ ++int zxdh_uresize_cq(struct ibv_cq *cq, int cqe) ++{ ++ struct zxdh_uvcontext *iwvctx; ++ struct zxdh_dev_attrs *dev_attrs; ++ struct zxdh_uresize_cq cmd = {}; ++ struct ib_uverbs_resize_cq_resp resp = {}; ++ struct zxdh_ureg_mr reg_mr_cmd = {}; ++ struct ib_uverbs_reg_mr_resp reg_mr_resp = {}; ++ struct zxdh_cq_buf *cq_buf = NULL; ++ struct zxdh_cqe *cq_base = NULL; ++ struct verbs_mr new_mr = {}; ++ struct zxdh_ucq *iwucq; ++ size_t cq_size; ++ __u32 cq_pages; ++ int cqe_needed; ++ int ret = 0; ++ ++ iwucq = container_of(cq, struct zxdh_ucq, verbs_cq.cq); ++ ++ iwvctx = container_of(cq->context, struct zxdh_uvcontext, ++ ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ ++ if (!(dev_attrs->feature_flags & ZXDH_FEATURE_CQ_RESIZE)) ++ return -EOPNOTSUPP; ++ ++ if (cqe > dev_attrs->max_hw_cq_size) ++ return -EINVAL; ++ cqe_needed = zxdh_cq_round_up(cqe); ++ ++ if (cqe_needed < ZXDH_U_MINCQ_SIZE) ++ cqe_needed = ZXDH_U_MINCQ_SIZE; ++ ++ if (cqe_needed * 2 <= dev_attrs->max_hw_cq_size) ++ cqe_needed = cqe_needed * 2; ++ if (cqe == cq->cqe) ++ return 0; ++ ++ cq_size = get_cq_total_bytes(cqe_needed); ++ cq_pages = cq_size >> ZXDH_HW_PAGE_SHIFT; ++ cq_base = (struct zxdh_cqe *)zxdh_alloc_hw_buf(cq_size); ++ if (!cq_base) ++ return -ENOMEM; ++ ++ memset(cq_base, 0, cq_size); ++ ++ cq_buf = malloc(sizeof(*cq_buf)); ++ if (!cq_buf) { ++ ret = -ENOMEM; ++ goto err_buf; ++ } ++ ++ ret = pthread_spin_lock(&iwucq->lock); ++ if (ret) ++ goto err_lock; ++ ++ new_mr.ibv_mr.pd = iwucq->vmr.ibv_mr.pd; ++ reg_mr_cmd.reg_type = ZXDH_MEMREG_TYPE_CQ; ++ reg_mr_cmd.cq_pages = cq_pages; ++ ret = ibv_cmd_reg_mr(new_mr.ibv_mr.pd, cq_base, cq_size, ++ (uintptr_t)cq_base, IBV_ACCESS_LOCAL_WRITE, ++ &new_mr, ®_mr_cmd.ibv_cmd, sizeof(reg_mr_cmd), ++ ®_mr_resp, sizeof(reg_mr_resp)); ++ if (ret) ++ goto err_dereg_mr; ++ ++ cmd.user_cq_buffer = (__u64)((uintptr_t)cq_base); ++ ret = ibv_cmd_resize_cq(&iwucq->verbs_cq.cq, cqe_needed, &cmd.ibv_cmd, ++ sizeof(cmd), &resp, sizeof(resp)); ++ if (ret) ++ goto err_resize; ++ ++ memcpy(&cq_buf->cq, &iwucq->cq, sizeof(cq_buf->cq)); ++ cq_buf->vmr = iwucq->vmr; ++ iwucq->vmr = new_mr; ++ zxdh_cq_resize(&iwucq->cq, cq_base, cqe_needed); ++ iwucq->verbs_cq.cq.cqe = cqe; ++ list_add_tail(&iwucq->resize_list, &cq_buf->list); ++ iwucq->resize_enable = true; ++ pthread_spin_unlock(&iwucq->lock); ++ ++ return ret; ++ ++err_resize: ++ ibv_cmd_dereg_mr(&new_mr); ++err_dereg_mr: ++ pthread_spin_unlock(&iwucq->lock); ++err_lock: ++ free(cq_buf); ++err_buf: ++ zxdh_free_hw_buf(cq_base, cq_size); ++ return ret; ++} ++ ++static void zxdh_srq_wqe_init(struct zxdh_usrq *iwusrq) ++{ ++ uint32_t i; ++ struct zxdh_srq *srq; ++ __le64 *wqe; ++ __u64 hdr; ++ ++ srq = &iwusrq->srq; ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s head:%d tail:%d\n", __func__, ++ srq->srq_ring.head, srq->srq_ring.tail); ++ for (i = srq->srq_ring.head; i < srq->srq_ring.tail; i++) { ++ wqe = zxdh_get_srq_wqe(srq, i); ++ ++ hdr = FIELD_PREP(ZXDHQPSRQ_NEXT_WQE_INDEX, (uint32_t)(i + 1)); ++ ++ udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */ ++ set_64bit_val(wqe, 0, hdr); ++ } ++} ++ ++static size_t zxdh_get_srq_queue_size(int srqdepth, __u8 srqshift) ++{ ++ return roundup((srqdepth << srqshift) * ZXDH_SRQ_WQE_MIN_SIZE, ZXDH_HW_PAGE_SIZE); ++} ++ ++static size_t zxdh_get_srq_list_size(size_t srq_size, __u8 srqshift) ++{ ++ return roundup((srq_size << srqshift) * sizeof(__u16), ZXDH_HW_PAGE_SIZE); ++} ++ ++static size_t zxdh_get_srq_db_size(void) ++{ ++ return 8 * sizeof(char); ++} ++ ++static size_t zxdh_get_total_srq_size(struct zxdh_usrq *iwusrq, int srqdepth, ++ size_t srq_size, __u8 srqshift) ++{ ++ size_t total_srq_queue_size; ++ size_t total_srq_list_size; ++ size_t total_srq_db_size; ++ size_t total_srq_size; ++ ++ total_srq_queue_size = zxdh_get_srq_queue_size(srqdepth, srqshift); ++ iwusrq->buf_size = total_srq_queue_size; ++ total_srq_list_size = zxdh_get_srq_list_size(srq_size, srqshift); ++ iwusrq->list_buf_size = total_srq_list_size; ++ total_srq_db_size = zxdh_get_srq_db_size(); ++ iwusrq->db_buf_size = total_srq_db_size; ++ total_srq_size = ++ total_srq_queue_size + total_srq_list_size + total_srq_db_size; ++ iwusrq->total_buf_size = total_srq_size; ++ zxdh_dbg( ++ ZXDH_DBG_SRQ, ++ "%s total_srq_queue_size:%ld total_srq_list_size:%ld total_srq_db_size:%ld srqdepth:%d\n", ++ __func__, total_srq_queue_size, total_srq_list_size, ++ total_srq_db_size, srqdepth); ++ ++ return total_srq_size; ++} ++ ++static int zxdh_alloc_srq_buf(struct zxdh_usrq *iwusrq, ++ struct zxdh_srq_init_info *info, ++ size_t total_srq_size) ++{ ++ struct zxdh_uvcontext *iwvctx; ++ iwvctx = container_of(iwusrq->ibv_srq.context, struct zxdh_uvcontext, ++ ibv_ctx.context); ++ info->srq_base = (struct zxdh_srq_wqe *)zxdh_alloc_hw_buf(total_srq_size); ++ if (!info->srq_base) ++ return -ENOMEM; ++ memset(info->srq_base, 0, total_srq_size); ++ info->srq_list_base = ++ (__le16 *)&info ++ ->srq_base[iwusrq->buf_size / ZXDH_SRQ_WQE_MIN_SIZE]; ++ if (iwvctx->srq_db == NULL) { ++ info->srq_db_base = ++ (__le64 *)&info->srq_list_base[iwusrq->list_buf_size / ++ (sizeof(__u16))]; ++ *(__le64 *)info->srq_db_base = ZXDH_SRQ_DB_INIT_VALUE; ++ } ++ zxdh_dbg(ZXDH_DBG_SRQ, ++ "%s srq_base:0x%p srq_list_base:0x%p srq_db_base:0x%p\n", ++ __func__, info->srq_base, info->srq_list_base, ++ info->srq_db_base); ++ return 0; ++} ++ ++static int zxdh_reg_srq_mr(struct ibv_pd *pd, struct zxdh_srq_init_info *info, ++ size_t total_srq_size, uint16_t srq_pages, ++ uint16_t srq_list_pages, struct zxdh_usrq *iwusrq) ++{ ++ struct zxdh_ureg_mr reg_mr_cmd = {}; ++ struct ib_uverbs_reg_mr_resp reg_mr_resp = {}; ++ int ret; ++ ++ reg_mr_cmd.reg_type = ZXDH_MEMREG_TYPE_SRQ; ++ reg_mr_cmd.srq_pages = srq_pages; ++ reg_mr_cmd.srq_list_pages = srq_list_pages; ++ ret = ibv_cmd_reg_mr(pd, info->srq_base, total_srq_size, ++ (uintptr_t)info->srq_base, IBV_ACCESS_LOCAL_WRITE, ++ &iwusrq->vmr, ®_mr_cmd.ibv_cmd, ++ sizeof(reg_mr_cmd), ®_mr_resp, ++ sizeof(reg_mr_resp)); ++ if (ret) ++ return ret; ++ ++ return 0; ++} ++ ++static int create_srq(struct ibv_pd *pd, struct zxdh_usrq *iwusrq, ++ struct ibv_srq_init_attr *attr, ++ struct zxdh_srq_init_info *info) ++{ ++ struct zxdh_ucreate_srq cmd = {}; ++ struct zxdh_ucreate_srq_resp resp = {}; ++ ++ struct zxdh_uvcontext *iwvctx; ++ int ret; ++ __u64 addr_offset = 0; ++ iwvctx = container_of(pd->context, struct zxdh_uvcontext, ++ ibv_ctx.context); ++ ++ cmd.user_wqe_bufs = (__u64)((uintptr_t)info->srq_base); ++ cmd.user_compl_ctx = (__u64)(uintptr_t)&iwusrq->srq; ++ cmd.user_wqe_list = (__u64)((uintptr_t)info->srq_list_base); ++ cmd.user_wqe_db = (__u64)((uintptr_t)info->srq_db_base); ++ ret = ibv_cmd_create_srq(pd, &iwusrq->ibv_srq, attr, &cmd.ibv_cmd, ++ sizeof(cmd), &resp.ibv_resp, ++ sizeof(struct zxdh_ucreate_srq_resp)); ++ if (ret) ++ return ret; ++ ++ iwusrq->srq_id = resp.srq_id; ++ info->srq_id = resp.srq_id; ++ info->srq_size = resp.actual_srq_size; ++ info->srq_list_size = resp.actual_srq_list_size; ++ if (iwvctx->srq_db != NULL) { ++ addr_offset = ++ (ZXDH_SYS_PAGE_SIZE == ZXDH_HW_64K_PAGE_SIZE) ? ++ (iwvctx->dev_attrs.srq_db_bar_off & (ZXDH_HW_64K_PAGE_SIZE - 1)) : ++ (iwvctx->dev_attrs.srq_db_bar_off & (ZXDH_HW_PAGE_SIZE - 1)); ++ info->base_srqn = resp.base_srqn; ++ ++ /* 防止 (srq_id - base_srqn) 发生下溢 */ ++ if (iwusrq->srq_id < info->base_srqn) { ++ zxdh_dbg(ZXDH_DBG_SRQ, ++ "%s invalid srq_id:%d < base_srqn:%d\n", ++ __func__, iwusrq->srq_id, info->base_srqn); ++ /* 已成功创建 SRQ，此处需清理 */ ++ ibv_cmd_destroy_srq(&iwusrq->ibv_srq); ++ return -EINVAL; ++ } ++ ++ info->srq_db_base = ++ (__le64 *)((__u8 *)iwvctx->srq_db + addr_offset + ++ (iwusrq->srq_id - info->base_srqn) * 8); ++ ++ set_64bit_val(info->srq_db_base, 0, ZXDH_SRQ_DB_INIT_VALUE); ++ zxdh_dbg(ZXDH_DBG_SRQ, ++ "%s info->srq_id:%d info->base_srqn:%d srq_db:0x%p\n", ++ __func__, info->srq_id, info->base_srqn, ++ iwvctx->srq_db); ++ } ++ zxdh_dbg( ++ ZXDH_DBG_SRQ, ++ "%s info->srq_id:%d info->srq_size:%d info->srq_list_size:%d srq_db_base:0x%p\n", ++ __func__, info->srq_id, info->srq_size, info->srq_list_size, ++ info->srq_db_base); ++ ++ return 0; ++} ++ ++/** ++ * zxdh_vmapped_srq - create resources for srq ++ * @iwusrq: srq struct for resources ++ * @pd: pd for the srq ++ * @attr: attributes of srq passed ++ * @resp: response back from create srq ++ * @srqdepth: depth of sq ++ * @info: info for initializing user level srq ++ */ ++static int zxdh_vmapped_srq(struct zxdh_usrq *iwusrq, struct ibv_pd *pd, ++ struct ibv_srq_init_attr *attr, int srqdepth, ++ __u8 srqshift, struct zxdh_srq_init_info *info) ++{ ++ size_t total_srq_size; ++ size_t srq_pages = 0; ++ size_t srq_list_pages = 0; ++ int ret; ++ ++ total_srq_size = ++ zxdh_get_total_srq_size(iwusrq, srqdepth, info->srq_size, srqshift); ++ srq_pages = iwusrq->buf_size >> ZXDH_HW_PAGE_SHIFT; ++ srq_list_pages = iwusrq->list_buf_size >> ZXDH_HW_PAGE_SHIFT; ++ ret = zxdh_alloc_srq_buf(iwusrq, info, total_srq_size); ++ if (ret) ++ return -ENOMEM; ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s srq_pages:%ld srq_list_pages:%ld\n", ++ __func__, srq_pages, srq_list_pages); ++ ++ ret = zxdh_reg_srq_mr(pd, info, total_srq_size, srq_pages, ++ srq_list_pages, iwusrq); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s %d ret:%d\n", __func__, __LINE__, ret); ++ if (ret) { ++ errno = ret; ++ goto err_dereg_srq_mr; ++ } ++ ret = create_srq(pd, iwusrq, attr, info); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s %d ret:%d\n", __func__, __LINE__, ret); ++ if (ret) ++ goto err_srq; ++ ++ return 0; ++err_srq: ++ ibv_cmd_dereg_mr(&iwusrq->vmr); ++err_dereg_srq_mr: ++ zxdh_free_hw_buf(info->srq_base, total_srq_size); ++ ++ return ret; ++} ++ ++/** ++ * zxdh_destroy_vmapped_srq - destroy resources for srq ++ * @iwusrq: srq struct for resources ++ */ ++static int zxdh_destroy_vmapped_srq(struct zxdh_usrq *iwusrq) ++{ ++ int ret; ++ ++ ret = ibv_cmd_destroy_srq(&iwusrq->ibv_srq); ++ if (ret) ++ return ret; ++ ++ ibv_cmd_dereg_mr(&iwusrq->vmr); ++ return 0; ++} ++ ++static int zxdh_check_srq_init_attr(struct ibv_srq_init_attr *srq_init_attr, ++ struct zxdh_dev_attrs *dev_attrs) ++{ ++ if ((srq_init_attr->attr.srq_limit > srq_init_attr->attr.max_wr) || ++ (srq_init_attr->attr.max_sge > dev_attrs->max_hw_wq_frags) || ++ (srq_init_attr->attr.max_wr > dev_attrs->max_hw_srq_wr)) { ++ return 1; ++ } ++ return 0; ++} ++ ++static int zxdh_init_iwusrq(struct zxdh_usrq *iwusrq, ++ struct ibv_srq_init_attr *srq_init_attr, ++ __u32 srqdepth, __u8 srqshift, ++ struct zxdh_srq_init_info *info, ++ struct zxdh_dev_attrs *dev_attrs) ++{ ++ info->srq_size = srqdepth; ++ iwusrq->max_wr = info->srq_size; ++ iwusrq->max_sge = srq_init_attr->attr.max_sge; ++ iwusrq->srq_limit = srq_init_attr->attr.srq_limit; ++ ++ srq_init_attr->attr.max_wr = info->srq_size; ++ info->dev_attrs = dev_attrs; ++ info->max_srq_frag_cnt = srq_init_attr->attr.max_sge; ++ info->srq_wrid_array = ++ calloc(info->srq_size, sizeof(*info->srq_wrid_array)); ++ if (info->srq_wrid_array == NULL) ++ return 1; ++ ++ return 0; ++} ++ ++/** ++ * zxdh_ucreate_srq - create srq on user app ++ * @pd: pd for the srq ++ * @srq_init_attr: attributes of the srq to be created (sizes, sge) ++ */ ++struct ibv_srq *zxdh_ucreate_srq(struct ibv_pd *pd, ++ struct ibv_srq_init_attr *srq_init_attr) ++{ ++ struct zxdh_srq_init_info info = {}; ++ struct zxdh_dev_attrs *dev_attrs; ++ struct zxdh_uvcontext *iwvctx; ++ __u32 srqdepth; ++ __u8 srqshift; ++ int status; ++ int ret; ++ struct zxdh_usrq *iwusrq; ++ __u32 usr_def_max_wr; ++ ++ iwvctx = container_of(pd->context, struct zxdh_uvcontext, ++ ibv_ctx.context); ++ dev_attrs = &iwvctx->dev_attrs; ++ ++ if ((zxdh_check_srq_init_attr(srq_init_attr, dev_attrs)) != 0) { ++ zxdh_dbg(ZXDH_DBG_SRQ, "zxdh_check_srq_init_attr failed\n"); ++ errno = EINVAL; ++ return NULL; ++ } ++ ++ if (iwvctx->is_qp_ext_mem_support && srq_init_attr->attr.max_sge > dev_attrs->max_hw_rq_sges) { ++ zxdh_dbg(ZXDH_DBG_SRQ, "zxdh_check_srq_init_attr failed, when use_blue_flame=true, srq max sge is 7.\n"); ++ errno = EINVAL; ++ return NULL; ++ } ++ ++ /* get shift count for maximum wqe size */ ++ zxdh_get_srq_wqe_shift(dev_attrs, srq_init_attr->attr.max_sge, ++ &srqshift); ++ ++ /* get RQ/SRQ depth (quanta)，minimum number of units in srq */ ++ status = zxdh_get_srqdepth(dev_attrs->max_hw_srq_quanta, ++ srq_init_attr->attr.max_wr, srqshift, ++ &srqdepth); ++ usr_def_max_wr = srq_init_attr->attr.max_wr; ++ zxdh_dbg( ++ ZXDH_DBG_SRQ, ++ "%s %d status:%d srqshift:%d srqdepth:%d dev_attrs->max_hw_srq_quanta:%d srq_init_attr->attr.max_wr:%d\n", ++ __func__, __LINE__, status, srqshift, srqdepth, ++ dev_attrs->max_hw_srq_quanta, srq_init_attr->attr.max_wr); ++ if (status != 0) { ++ zxdh_dbg(ZXDH_DBG_SRQ, "zxdh_get_srqdepth failed\n"); ++ errno = EINVAL; ++ return NULL; ++ } ++ iwusrq = memalign(1024, sizeof(*iwusrq)); ++ if (!iwusrq) ++ return NULL; ++ memset(iwusrq, 0, sizeof(*iwusrq)); ++ if (pthread_spin_init(&iwusrq->lock, PTHREAD_PROCESS_PRIVATE) != 0) ++ goto err_free_srq; ++ ++ if (zxdh_init_iwusrq(iwusrq, srq_init_attr, srqdepth, srqshift, &info, ++ dev_attrs)) { ++ zxdh_dbg(ZXDH_DBG_SRQ, "calloc srq_wrid_array failed\n"); ++ goto err_srq_wrid_array; ++ } ++ iwusrq->ibv_srq.context = pd->context; ++ status = zxdh_vmapped_srq(iwusrq, pd, srq_init_attr, srqdepth, srqshift, &info); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s %d status:%d\n", __func__, __LINE__, status); ++ if (status) { ++ zxdh_dbg(ZXDH_DBG_SRQ, "zxdh_vmapped_srq failed\n"); ++ errno = status; ++ goto err_vmapped_srq; ++ } ++ ++ status = zxdh_srq_init(&iwusrq->srq, &info); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s %d status:%d\n", __func__, __LINE__, status); ++ if (status) { ++ zxdh_dbg(ZXDH_DBG_SRQ, "zxdh_srq_init failed\n"); ++ errno = EINVAL; ++ goto err_free_srq_init; ++ } ++ zxdh_srq_wqe_init(iwusrq); ++ ++ //srq_init_attr->attr.max_wr = (srqdepth - ZXDH_SRQ_RSVD) >> srqshift; ++ srq_init_attr->attr.max_wr = usr_def_max_wr; ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s iwusrq->srq_id:%d info.srq_size:%d\n", ++ __func__, iwusrq->srq_id, info.srq_size); ++ return &iwusrq->ibv_srq; ++ ++err_free_srq_init: ++ zxdh_destroy_vmapped_srq(iwusrq); ++ zxdh_free_hw_buf(info.srq_base, iwusrq->total_buf_size); ++err_vmapped_srq: ++ free(info.srq_wrid_array); ++err_srq_wrid_array: ++ ret = pthread_spin_destroy(&iwusrq->lock); ++ if (ret) ++ errno = EINVAL; ++err_free_srq: ++ free(iwusrq); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s %d\n", __func__, __LINE__); ++ return NULL; ++} ++ ++/** ++ * zxdh_udestroy_srq - destroy srq on user app ++ * @srq: srq to destroy ++ */ ++int zxdh_udestroy_srq(struct ibv_srq *srq) ++{ ++ struct zxdh_usrq *iwusrq; ++ int ret; ++ ++ iwusrq = container_of(srq, struct zxdh_usrq, ibv_srq); ++ ret = pthread_spin_destroy(&iwusrq->lock); ++ if (ret) ++ goto err; ++ ++ ret = zxdh_destroy_vmapped_srq(iwusrq); ++ if (ret) ++ goto err; ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s iwusrq->srq_id:%d\n", __func__, ++ iwusrq->srq_id); ++ zxdh_free_hw_buf(iwusrq->srq.srq_base, iwusrq->total_buf_size); ++ free(iwusrq->srq.srq_wrid_array); ++ free(iwusrq); ++ ++ return 0; ++ ++err: ++ return ret; ++} ++ ++/** ++ * zxdh_umodify_srq - modify srq on user app ++ * @srq: srq to destroy ++ */ ++int zxdh_umodify_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr, ++ int srq_attr_mask) ++{ ++ struct ibv_modify_srq cmd; ++ struct zxdh_usrq *iwusrq; ++ int ret; ++ ++ iwusrq = container_of(srq, struct zxdh_usrq, ibv_srq); ++ ret = ibv_cmd_modify_srq(srq, srq_attr, srq_attr_mask, &cmd, ++ sizeof(cmd)); ++ if (ret == 0) ++ iwusrq->srq_limit = srq_attr->srq_limit; ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s iwusrq->srq_id:%d srq_attr->srq_limit:%d\n", ++ __func__, iwusrq->srq_id, srq_attr->srq_limit); ++ return ret; ++} ++ ++/** ++ * zxdh_uquery_srq - query srq on user app ++ * @srq: srq to query ++ * @srq_attr: attributes of the srq to be query ++ */ ++int zxdh_uquery_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr) ++{ ++ struct ibv_query_srq cmd; ++ ++ return ibv_cmd_query_srq(srq, srq_attr, &cmd, sizeof(cmd)); ++} ++ ++static int zxdh_check_srq_valid(struct ibv_recv_wr *recv_wr, ++ struct zxdh_usrq *iwusrq, struct zxdh_srq *srq) ++{ ++ if (unlikely(recv_wr->num_sge > iwusrq->max_sge)) ++ return ZXDH_ERR_INVALID_FRAG_COUNT; ++ ++ if (unlikely(srq->srq_ring.head == srq->srq_ring.tail)) ++ return ZXDH_ERR_RING_FULL; ++ ++ return 0; ++} ++ ++static void zxdh_fill_srq_wqe(struct zxdh_usrq *iwusrq, struct zxdh_srq *srq, ++ __le64 *wqe_64, struct ibv_recv_wr *recv_wr) ++{ ++ __u32 byte_off; ++ int i; ++ ++ for (i = 0, byte_off = ZXDH_SRQ_FRAG_BYTESIZE; ++ i < recv_wr->num_sge && ++ byte_off + ZXDH_SRQ_FRAG_BYTESIZE < UINT32_MAX; ++ i++) { ++ set_64bit_val(wqe_64, byte_off, recv_wr->sg_list[i].addr); ++ set_64bit_val(wqe_64, byte_off + 8, ++ FIELD_PREP(ZXDHQPSRQ_FRAG_LEN, ++ recv_wr->sg_list[i].length) | ++ FIELD_PREP(ZXDHQPSRQ_FRAG_STAG, ++ recv_wr->sg_list[i].lkey)); ++ byte_off += ZXDH_SRQ_FRAG_BYTESIZE; ++ } ++ ++ if ((recv_wr->num_sge < iwusrq->max_sge) || (recv_wr->num_sge == 0)) { ++ set_64bit_val(wqe_64, byte_off, 0); ++ set_64bit_val(wqe_64, byte_off + 8, ++ FIELD_PREP(ZXDHQPSRQ_FRAG_LEN, 0) | ++ FIELD_PREP(ZXDHQPSRQ_FRAG_STAG, ++ ZXDH_SRQ_INVALID_LKEY)); ++ } ++ ++ set_64bit_val(wqe_64, 8, ((uint64_t)iwusrq->srq_id) << 32); ++ ++ __u64 hdr = FIELD_PREP(ZXDHQPSRQ_RSV, 0) | ++ FIELD_PREP(ZXDHQPSRQ_VALID_SGE_NUM, recv_wr->num_sge) | ++ FIELD_PREP(ZXDHQPSRQ_SIGNATURE, 0) | ++ FIELD_PREP(ZXDHQPSRQ_NEXT_WQE_INDEX, srq->srq_ring.head); ++ ++ udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */ ++ set_64bit_val(wqe_64, 0, hdr); ++ ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s wqe_64[0]:0x%llx\n", __func__, wqe_64[0]); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s wqe_64[1]:0x%llx\n", __func__, wqe_64[1]); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s wqe_64[2]:0x%llx\n", __func__, wqe_64[2]); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s wqe_64[3]:0x%llx\n", __func__, wqe_64[3]); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s wqe_64[4]:0x%llx\n", __func__, wqe_64[4]); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s wqe_64[5]:0x%llx\n", __func__, wqe_64[5]); ++} ++ ++static void zxdh_get_wqe_index(struct zxdh_srq *srq, __le16 *wqe_16, __u16 *buf, ++ __u16 nreq, __u16 *idx) ++{ ++ int i; ++ ++ for (i = 0; i < nreq; i++) { ++ wqe_16 = zxdh_get_srq_list_wqe(srq, idx); ++ udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */ ++ set_16bit_val(wqe_16, 0, buf[i]); ++ } ++} ++ ++static void zxdh_update_srq_db_base(struct zxdh_usrq *iwusrq, __u16 idx) ++{ ++ __u64 hdr = FIELD_PREP(ZXDH_SRQ_PARITY_SIGN, ++ iwusrq->srq.srq_list_polarity) | ++ FIELD_PREP(ZXDH_SRQ_SW_SRQ_HEAD, idx); ++ ++ udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */ ++ set_64bit_val(iwusrq->srq.srq_db_base, 0, hdr); ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s srq_db_base(hdr):0x%llx\n", __func__, hdr); ++} ++ ++/** ++ * zxdh_upost_srq_recv - post srq recv on user app ++ * @srq: srq to post recv ++ * @recv_wr: a list of work requests to post on the receive queue ++ * @bad_recv_wr: pointer to first rejected wr ++ */ ++int zxdh_upost_srq_recv(struct ibv_srq *srq, struct ibv_recv_wr *recv_wr, ++ struct ibv_recv_wr **bad_recv_wr) ++{ ++ struct zxdh_usrq *iwusrq; ++ struct zxdh_srq *hw_srq; ++ __le16 *wqe_16; ++ __le64 *wqe_64; ++ __u64 temp_val; ++ int err = 0; ++ errno = 0; ++ int nreq; ++ __u16 *buf; ++ size_t buf_size; ++ __u16 idx = 0; ++ ++ iwusrq = container_of(srq, struct zxdh_usrq, ibv_srq); ++ hw_srq = &iwusrq->srq; ++ pthread_spin_lock(&iwusrq->lock); ++ buf_size = iwusrq->max_wr * sizeof(__u16); ++ buf = malloc(buf_size); ++ if (buf == NULL) { ++ zxdh_status_to_errno_log(ZXDH_ERR_NO_MEMORY,"srq malloc buf_size failed, srq_id=%d", iwusrq->srq_id); ++ goto out; ++ } ++ ++ for (nreq = 0; recv_wr; nreq++, recv_wr = recv_wr->next) { ++ err = zxdh_check_srq_valid(recv_wr, iwusrq, hw_srq); ++ if (err){ ++ zxdh_status_to_errno_log(err,"srq check valid failed, srq_id=%d", iwusrq->srq_id); ++ break; ++ } ++ ++ iwusrq->srq.srq_wrid_array[hw_srq->srq_ring.head] = ++ recv_wr->wr_id; ++ buf[nreq] = hw_srq->srq_ring.head; ++ wqe_64 = zxdh_get_srq_wqe(hw_srq, hw_srq->srq_ring.head); ++ get_64bit_val(wqe_64, 0, &temp_val); ++ hw_srq->srq_ring.head = ++ (__u16)FIELD_GET(ZXDHQPSRQ_NEXT_WQE_INDEX, temp_val); ++ zxdh_fill_srq_wqe(iwusrq, hw_srq, wqe_64, recv_wr); ++ } ++ ++ zxdh_dbg(ZXDH_DBG_SRQ, "%s nreq:%d err:%d iwusrq->srq_id:%d\n", ++ __func__, nreq, err, iwusrq->srq_id); ++ ++ if (errno == 0) { ++ zxdh_get_wqe_index(hw_srq, wqe_16, buf, nreq, &idx); ++ zxdh_update_srq_db_base(iwusrq, idx); ++ } ++out: ++ err = errno; ++ pthread_spin_unlock(&iwusrq->lock); ++ if (err) ++ *bad_recv_wr = recv_wr; ++ if (buf) ++ free(buf); ++ return err; ++} ++ ++/** ++ * zxdh_uget_srq_num - get srq num on user app ++ * @srq: srq to get num ++ * @srq_num: to get srq num ++ */ ++int zxdh_uget_srq_num(struct ibv_srq *srq, uint32_t *srq_num) ++{ ++ struct zxdh_usrq *iwusrq; ++ ++ iwusrq = container_of(srq, struct zxdh_usrq, ibv_srq); ++ ++ *srq_num = iwusrq->srq_id; ++ return 0; ++} ++ ++void zxdh_set_debug_mask(void) ++{ ++ char *env; ++ ++ env = getenv("ZXDH_DEBUG_MASK"); ++ if (env) ++ zxdh_debug_mask = strtol(env, NULL, 0); ++} ++ ++int zxdh_get_write_imm_split_switch(void) ++{ ++ char *env; ++ env = getenv("ZXDH_WRITE_IMM_SPILT_ENABLE"); ++ return (env != NULL) ? atoi(env) : 0; ++} +diff -ruN baseline/providers/zrdma/zxdh_verbs.h b/providers/zrdma/zxdh_verbs.h +--- a/providers/zrdma/zxdh_verbs.h 2026-08-06 15:37:55.360863311 +0800 ++++ b/providers/zrdma/zxdh_verbs.h 2026-08-06 15:29:37.431688351 +0800 +@@ -3,7 +3,7 @@ + #ifndef ZXDH_VERBS_H + #define ZXDH_VERBS_H + #include "zxdh_defs.h" +- ++#include + #define zxdh_handle void * + #define zxdh_adapter_handle zxdh_handle + #define zxdh_qp_handle zxdh_handle +@@ -45,6 +45,7 @@ + #define ZXDH_OP_TYPE_UD_SEND_WITH_IMM 0x0b + #define ZXDH_OP_TYPE_REC 0x3e + #define ZXDH_OP_TYPE_REC_IMM 0x3f ++#define ZXDH_OP_TYPE_REC_WRITE_IMM 0x4f + + #define ZXDH_FLUSH_MAJOR_ERR 1 + #define ZXDH_RETRY_ACK_MAJOR_ERR 0x8 +@@ -71,47 +72,13 @@ + + #define ZXDH_WQEALLOC_WQE_DESC_INDEX GENMASK(31, 20) + +-enum zxdh_device_caps_const { +- ZXDH_WQE_SIZE = 4, +- ZXDH_SRQE_SIZE = 2, +- ZXDH_CQP_WQE_SIZE = 8, +- ZXDH_CQE_SIZE = 8, +- ZXDH_EXTENDED_CQE_SIZE = 8, +- ZXDH_AEQE_SIZE = 2, +- ZXDH_CEQE_SIZE = 1, +- ZXDH_CQP_CTX_SIZE = 8, +- ZXDH_SHADOW_AREA_SIZE = 8, +- ZXDH_GATHER_STATS_BUF_SIZE = 1024, +- ZXDH_MIN_IW_QP_ID = 0, +- ZXDH_QUERY_FPM_BUF_SIZE = 176, +- ZXDH_COMMIT_FPM_BUF_SIZE = 176, +- ZXDH_MAX_IW_QP_ID = 262143, +- ZXDH_MIN_CEQID = 0, +- ZXDH_MAX_CEQID = 1023, +- ZXDH_CEQ_MAX_COUNT = ZXDH_MAX_CEQID + 1, +- ZXDH_MIN_CQID = 0, +- ZXDH_MAX_CQID = 524287, +- ZXDH_MIN_AEQ_ENTRIES = 1, +- ZXDH_MAX_AEQ_ENTRIES = 524287, +- ZXDH_MIN_CEQ_ENTRIES = 1, +- ZXDH_MAX_CEQ_ENTRIES = 262143, +- ZXDH_MIN_CQ_SIZE = 1, +- ZXDH_MAX_CQ_SIZE = 1048575, +- ZXDH_DB_ID_ZERO = 0, +- ZXDH_MAX_WQ_FRAGMENT_COUNT = 13, +- ZXDH_MAX_SGE_RD = 13, +- ZXDH_MAX_OUTBOUND_MSG_SIZE = 2147483647, +- ZXDH_MAX_INBOUND_MSG_SIZE = 2147483647, +- ZXDH_MAX_PUSH_PAGE_COUNT = 1024, +- ZXDH_MAX_PE_ENA_VF_COUNT = 32, +- ZXDH_MAX_VF_FPM_ID = 47, +- ZXDH_MAX_SQ_PAYLOAD_SIZE = 2147483648, +- ZXDH_MAX_INLINE_DATA_SIZE = 217, +- ZXDH_MAX_WQ_ENTRIES = 32768, +- ZXDH_Q2_BUF_SIZE = 256, +- ZXDH_QP_CTX_SIZE = 256, +- ZXDH_MAX_PDS = 262144, +-}; ++#define ZXDH_SRQE_SIZE 2 ++#define ZXDH_CQE_SIZE 8 ++#define ZXDH_EXTENDED_CQE_SIZE 8 ++#define ZXDH_MAX_INLINE_DATA_SIZE 217 ++#define ZXDH_MAX_SQ_PAYLOAD_SIZE 2147483648 ++#define ZXDH_MIN_CQ_SIZE 1 ++#define ZXDH_MAX_CQ_SIZE 2097152 + + enum zxdh_addressing_type { + ZXDH_ADDR_TYPE_ZERO_BASED = 0, +@@ -176,10 +143,124 @@ + + enum zxdh_page_size { + ZXDH_PAGE_SIZE_4K = 0, ++ ZXDH_PAGE_SIZE_64K = 4, + ZXDH_PAGE_SIZE_2M = 9, + ZXDH_PAGE_SIZE_1G = 18, + }; + ++enum zxdh_rdmatx_parse_top_err { ++ ZXDH_TX_PARSE_TOP_AXI_ERR = 0x1, ++ ZXDH_TX_PARSE_TOP_WQE_FLUSH = 0x10001, ++ ZXDH_TX_PARSE_TOP_ORD_ERR = 0x20020, ++ ZXDH_TX_PARSE_TOP_OPCODE_ERR_FLAG = 0x20021, ++ ZXDH_TX_PARSE_TOP_CQP_STATE_AXI_ERR = 0x20022, ++ ZXDH_TX_PARSE_TOP_WQE_LEN_ERR = 0x20023, ++ ZXDH_TX_PARSE_TOP_DATA_LEN_ERR = 0x20024, ++ ZXDH_TX_PARSE_TOP_AH_VALID0_ERR = 0x20025, ++ ZXDH_TX_PARSE_TOP_UD_PDINDEX_ERR = 0x20026, ++ ZXDH_TX_PARSE_TOP_QP_STATE_ERR = 0x20027, ++ ZXDH_TX_PARSE_TOP_SERVICE_TYPE_ERR = 0x20028, ++ ZXDH_TX_PARSE_TOP_UD_PAYLOAD_ERR = 0x20029, ++ ZXDH_TX_PARSE_TOP_WQE_LEN0_ERR = 0x2002a, ++ ZXDH_TX_PARSE_TOP_WQE_DEFICIENT_CLR_ERR = 0x2002b, ++ ZXDH_TX_PARSE_TOP_IMMDT_ERR = 0x2002c, ++ ZXDH_TX_PARSE_TOP_FRAGMENT_LENGTH_ERR = 0x2009f, ++ ZXDH_TX_PARSE_TOP_MRTE_STATE_ERR = 0x90091, ++ ZXDH_TX_PARSE_TOP_QP_CHECK_ERR = 0x90092, ++ ZXDH_TX_PARSE_TOP_PD_CHECK_ERR = 0x90093, ++ ZXDH_TX_PARSE_TOP_LKEY_CHECK_ERR = 0x90094, ++ ZXDH_TX_PARSE_TOP_STAG_INDEX_CHECK_ERR = 0x90095, ++ ZXDH_TX_PARSE_TOP_VADDR_LEN_CHECK_ERR = 0x90096, ++ ZXDH_TX_PARSE_TOP_ACCESS_RIGHT_CHECK_ERR = 0x90097, ++ ZXDH_TX_PARSE_TOP_STAG_INDEX_CHECK_ZERO_ERR = 0x90098, ++}; ++ ++enum zxdh_rdmatx_ack_sys_top_err { ++ ZXDH_TX_ACK_SYS_TOP_NVME_INDEX_ERR = 0x30030, ++ ZXDH_TX_ACK_SYS_TOP_NVME_NOF_QID_ERR = 0x30031, ++ ZXDH_TX_ACK_SYS_TOP_NVME_NOF_PD_INDEX_ERR = 0x30032, ++ ZXDH_TX_ACK_SYS_TOP_NVME_LENGTH_ERR = 0x30033, ++ ZXDH_TX_ACK_SYS_TOP_NVME_KEY_ERR = 0x30034, ++ ZXDH_TX_ACK_SYS_TOP_NVME_ACCESS_ERR = 0x30035, ++ ZXDH_TX_ACK_SYS_TOP_MRTE_STATE_ERR = 0x50091, ++ ZXDH_TX_ACK_SYS_TOP_QP_CHECK_ERR = 0x50092, ++ ZXDH_TX_ACK_SYS_TOP_PD_CHECK_ERR = 0x50093, ++ ZXDH_TX_ACK_SYS_TOP_LKEY_CHECK_ERR = 0x50094, ++ ZXDH_TX_ACK_SYS_TOP_STAG_INDEX_CHECK_ERR = 0x50095, ++ ZXDH_TX_ACK_SYS_TOP_VADDR_LEN_CHECK_ERR = 0x50096, ++ ZXDH_TX_ACK_SYS_TOP_ACCESS_RIGHT_CHECK_ERR = 0x50097, ++ ZXDH_TX_ACK_SYS_TOP_STAG_INDEX_CHECK_ZERO_ERR = 0x50098, ++ ZXDH_TX_ACK_SYS_TOP_LOC_LEN_ERR = 0x600c0, ++ ZXDH_TX_ACK_SYS_TOP_NAK_INVALID_REQ = 0x700d0, ++ ZXDH_TX_ACK_SYS_TOP_NAK_REMOTE_ACCESS_ERR = 0x700d1, ++ ZXDH_TX_ACK_SYS_TOP_NAK_REMOTE_OPERATIONAL_ERR = 0x700d2, ++ ZXDH_TX_ACK_SYS_TOP_NAK_RETRY_LIMIT = 0x800f1, ++ ZXDH_TX_ACK_SYS_TOP_READ_RETRY_LIMIT = 0x800f2, ++ ZXDH_TX_ACK_SYS_TOP_TIMEOUT_RETRY_LIMIT = 0x800f3, ++ ZXDH_TX_ACK_SYS_TOP_RNR_RETRY_LIMIT = 0x800f4, ++}; ++ ++enum zxdh_rdmatx_window_top_err { ++ ZXDH_TX_WINDOW_TOP_WINDOW_NO_ENTRY = 0x800f5, ++ ZXDH_TX_WINDOW_TOP_WINDOW_BACK_MSN = 0x800f6, ++ ZXDH_TX_WINDOW_TOP_WINDOW_SMALL_MSN = 0x800f7, ++}; ++ ++enum zxdh_rdmatx_doorbell_mgr_err { ++ ZXDH_TX_DOORBELL_MGR_INDEX_CHECK_ERROR = 0x30036, ++ ZXDH_TX_DOORBELL_MGR_QID_CHECK_ERROR = 0x30037, ++ ZXDH_TX_DOORBELL_MGR_PD_INDEX_CHECK_ERROR = 0x30038, ++ ZXDH_TX_DOORBELL_MGR_LENGTH_CHECK_ERROR = 0x30039, ++ ZXDH_TX_DOORBELL_MGR_KEY_CHECK_ERROR = 0x3003a, ++ ZXDH_TX_DOORBELL_MGR_ACCESS_CHECK_ERROR = 0x3003b, ++}; ++ ++enum zxdh_rdmarx_err { ++ ZXDH_RX_CQP_FLUSH = 0x12, ++ ZXDH_RX_FIRST_PACKET_ERR = 0x4f, ++ ZXDH_RX_INVALID_OPCODE = 0x50, ++ ZXDH_RX_ORDER_ERR = 0x51, ++ ZXDH_RX_LEN_ERR = 0x52, ++ ZXDH_RX_SQR_STATE_ERR = 0x53, ++ ZXDH_RX_WQE_SIGN_ERR = 0x54, ++ ZXDH_RX_WQE_LEN_ERR = 0x55, ++ ZXDH_RX_SQR_WATER_LEVEL_ERR = 0x80, ++ ZXDH_RX_SRQ_AXI_RESP_ERR = 0xb1, ++ ZXDH_RX_CQ_OVERFLOW_ERR = 0x76, ++ ZXDH_RX_QP_CQ_OVERFLOW_ERR = 0x78, ++ ZXDH_RX_CQ_STATE_ERR = 0x7a, ++ ZXDH_RX_CQ_AXI_ERR = 0x7b, ++ ZXDH_RX_QP_CQ_AXI_ERR = 0x7c, ++ ZXDH_RX_NOF_IOQ_ERR = 0x70, ++ ZXDH_RX_NOF_PDNUM_ERR = 0x71, ++ ZXDH_RX_NOF_LEN_ERR = 0x72, ++ ZXDH_RX_NOF_RKEY_ERR = 0x73, ++ ZXDH_RX_NOF_ACC_ERR = 0x74, ++ ZXDH_RX_IRD_OVF = 0x77, ++ ZXDH_RX_MR_MW_STATE_FREE_ERR = 0x90, ++ ZXDH_RX_MR_MW_STATE_INVALID_ERR = 0x91, ++ ZXDH_RX_TYPE2B_MW_QPN_CHECK_ERR = 0x92, ++ ZXDH_RX_MR_MW_PD_CHECK_ERR = 0x93, ++ ZXDH_RX_MR_MW_KEY_CHECK_ERR = 0x94, ++ ZXDH_RX_MR_MW_STAG_INDEX_CHECK_ERR = 0x95, ++ ZXDH_RX_MR_MW_BOUNDARY_CHECK_ERR = 0x96, ++ ZXDH_RX_MR_MW_ACCESS_CHECK_ERR = 0x97, ++ ZXDH_RX_MR_MW_0STAG_INDEX_CHECK_ERR = 0x98, ++ ZXDH_RX_MW_STATE_INVALID_ERR = 0x99, ++ ZXDH_RX_MW_PD_CHECK_ERR = 0x9a, ++ ZXDH_RX_MW_RKEY_CHECK_ERR = 0x9b, ++ ZXDH_RX_TYPE2BMW_QPN_CHECK_ERR = 0x9c, ++ ZXDH_RX_MW_STAG_INDEX_CHECK_ERR = 0x9d, ++ ZXDH_RX_MW_SHARE_MR_CHECK_ERR = 0x9e, ++ ZXDH_RX_MW_TYPE1_CHECK_ERR = 0x9f, ++ ZXDH_RX_MR_PD_CHECK_ERR = 0xa0, ++ ZXDH_RX_MR_RKEY_CHECK_ERR = 0xa1, ++ ZXDH_RX_MR_SHARE_MR_CHECK_ERR = 0xa4, ++ ZXDH_RX_MR_BOND_MW_NUM_CHECK_ERR = 0xa5, ++ ZXDH_RX_MR_CANBE_R_INVALID_CHECK_ERR = 0xa6, ++ ZXDH_RX_AXI_RESP_ERR = 0xb0, ++}; ++ + struct zxdh_qp; + struct zxdh_cq; + struct zxdh_qp_init_info; +@@ -195,6 +276,7 @@ + __u32 head; + __u32 tail; + __u32 size; ++ __u32 used_quanta_cnt; /* 新增：已使用的量子数计数 */ + }; + + struct zxdh_cqe { +@@ -327,6 +409,7 @@ + bool signaled, bool post_sq); + enum zxdh_status_code zxdh_post_receive(struct zxdh_qp *qp, + struct zxdh_post_rq_info *info); ++void read_wqe_ctrl_hdr_coherent_if_need(struct zxdh_qp *qp, __le64 *wqe); + void zxdh_qp_post_wr(struct zxdh_qp *qp); + void zxdh_qp_set_shadow_area(struct zxdh_qp *qp); + enum zxdh_status_code zxdh_rdma_read(struct zxdh_qp *qp, +@@ -394,8 +477,13 @@ + __u32 max_hw_cq_size; + __u16 max_hw_sq_chunk; + __u32 max_hw_srq_wr; +- __u8 hw_rev; + __u8 db_addr_type; ++ __u8 chip_rev; ++ __u16 rdma_tool_flags; ++ __aligned_u64 sq_db_bar_off; ++ __aligned_u64 cq_db_bar_off; ++ __aligned_u64 srq_db_bar_off; ++ __u32 max_hw_rq_sges; + }; + + struct zxdh_hw_attrs { +@@ -461,6 +549,7 @@ + void *back_qp; + zxdh_sgl split_sg_list; + pthread_spinlock_t *lock; ++ pthread_spinlock_t *quanta_lock; + __u16 rwqe_signature; + __u8 dbg_rq_flushed; + __u8 sq_flush_seen; +@@ -472,6 +561,14 @@ + __u32 qp_last_ack_qsn; + __u8 cqe_retry_cnt; + __u8 qp_reset_cnt; ++ bool credit_set; ++ __u32 last_wqe; ++ __u8 is_ext_qp; ++ struct zxdh_qp_sq_quanta *ext_sq_base_va; ++ struct zxdh_qp_rq_quanta *ext_rq_base_va; ++ __u8 ext_buf_mode; ++ __u32 last_sqe_idx; /* 新增：上次poll的SQ WQE索引 */ ++ __u32 last_rqe_idx; /* 新增：上次poll的RQ WQE索引 */ + }; + + struct zxdh_cq { +@@ -529,6 +626,13 @@ + __u8 type; + int abi_ver; + bool legacy_mode; ++ __u32 sqdepth; ++ __u32 rqdepth; ++ __u8 sqshift; ++ __u8 rqshift; ++ __u8 is_qp_ext_mem_support; ++ __u32 *ext_sq_db; ++ void *ext_qp_buff_base; + }; + + struct zxdh_cq_init_info { +@@ -555,6 +659,7 @@ + __u32 srq_db_size; + __u32 max_srq_frag_cnt; + __u32 srq_limit; ++ __u32 base_srqn; + }; + + struct zxdh_wqe_srq_next_sge { +@@ -574,6 +679,23 @@ + __le64 elem[ZXDH_SRQE_SIZE]; + }; + ++#if defined(__cplusplus) && (__cplusplus < 202002L) ++// C++20 之前版本：使用顺序初始化 ++static const unsigned int zxdh_roce_mtu[] = { 256, 512, 1024, 2048, 4096 }; ++#else ++// C99 或 C++20+：使用指定初始化 ++static const unsigned int zxdh_roce_mtu[] = { [IBV_MTU_256] = 256, ++ [IBV_MTU_512] = 512, ++ [IBV_MTU_1024] = 1024, ++ [IBV_MTU_2048] = 2048, ++ [IBV_MTU_4096] = 4096 }; ++#endif ++ ++static inline unsigned int mtu_enum_to_int(enum ibv_mtu mtu) ++{ ++ return zxdh_roce_mtu[mtu]; ++} ++ + __le64 *zxdh_qp_get_next_send_wqe(struct zxdh_qp *qp, __u32 *wqe_idx, + __u16 quanta, __u32 total_size, + struct zxdh_post_sq_info *info); +@@ -581,22 +703,27 @@ + void zxdh_clean_cq(void *q, struct zxdh_cq *cq); + enum zxdh_status_code zxdh_nop(struct zxdh_qp *qp, __u64 wr_id, bool signaled, + bool post_sq); +-enum zxdh_status_code zxdh_fragcnt_to_quanta_sq(__u32 frag_cnt, __u16 *quanta); + enum zxdh_status_code zxdh_fragcnt_to_wqesize_rq(__u32 frag_cnt, + __u16 *wqe_size); + void zxdh_get_sq_wqe_shift(__u32 sge, __u32 inline_data, __u8 *shift); ++ + void zxdh_get_rq_wqe_shift(__u32 sge, __u8 *shift); + enum zxdh_status_code zxdh_get_sqdepth(struct zxdh_dev_attrs *dev_attrs, + __u32 sq_size, __u8 shift, + __u32 *wqdepth); ++enum zxdh_status_code zxdh_devx_get_sqdepth(struct zxdh_dev_attrs *dev_attrs, ++ __u32 sq_size, __u8 shift, ++ __u32 *wqdepth); + enum zxdh_status_code zxdh_get_rqdepth(struct zxdh_dev_attrs *dev_attrs, + __u32 rq_size, __u8 shift, + __u32 *wqdepth); ++enum zxdh_status_code zxdh_devx_get_rqdepth(struct zxdh_dev_attrs *dev_attrs, ++ __u32 sq_size, __u8 shift, ++ __u32 *wqdepth); + int zxdh_qp_round_up(__u32 wqdepth); + int zxdh_cq_round_up(__u32 wqdepth); + void zxdh_qp_push_wqe(struct zxdh_qp *qp, __le64 *wqe, __u16 quanta, + __u32 wqe_idx, bool post_sq); +-void zxdh_clr_wqes(struct zxdh_qp *qp, __u32 qp_wqe_idx); + + void zxdh_get_srq_wqe_shift(struct zxdh_dev_attrs *dev_attrs, __u32 sge, + __u8 *shift); +diff -ruN baseline/providers/zrdma/zxdh_zrdma.c b/providers/zrdma/zxdh_zrdma.c +--- a/providers/zrdma/zxdh_zrdma.c 1970-01-01 08:00:00.000000000 +0800 ++++ b/providers/zrdma/zxdh_zrdma.c 2026-08-06 15:29:37.198672348 +0800 +@@ -0,0 +1,344 @@ ++// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB ++/* Copyright (c) 2024 ZTE Corporation. All rights reserved. */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "zxdh_devids.h" ++#include "zxdh_zrdma.h" ++#include "zxdh_abi.h" ++#include "private_verbs_cmd.h" ++ ++ ++#define ZXDH_HCA(v, d) VERBS_PCI_MATCH(v, d, NULL) ++static const struct verbs_match_ent hca_table[] = { ++ VERBS_DRIVER_ID(RDMA_DRIVER_ZXDH), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_EVB, ZXDH_DEV_ID_ADAPTIVE_EVB_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_EVB, ZXDH_DEV_ID_ADAPTIVE_EVB_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312, ZXDH_DEV_ID_ADAPTIVE_E312_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312, ZXDH_DEV_ID_ADAPTIVE_E312_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E310, ZXDH_DEV_ID_ADAPTIVE_E310_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E310, ZXDH_DEV_ID_ADAPTIVE_E310_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E310_RDMA, ++ ZXDH_DEV_ID_ADAPTIVE_E310_RDMA_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E310_RDMA, ++ ZXDH_DEV_ID_ADAPTIVE_E310_RDMA_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E316, ZXDH_DEV_ID_ADAPTIVE_E316_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E316, ZXDH_DEV_ID_ADAPTIVE_E316_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E318, ZXDH_DEV_ID_ADAPTIVE_E318_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E318, ZXDH_DEV_ID_ADAPTIVE_E318_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_X512, ZXDH_DEV_ID_ADAPTIVE_X512_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_X512, ZXDH_DEV_ID_ADAPTIVE_X512_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312_TY_CLOUD, ++ ZXDH_DEV_ID_ADAPTIVE_E312_TY_CLOUD_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312_TY_CLOUD, ++ ZXDH_DEV_ID_ADAPTIVE_E312_TY_CLOUD_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E310_TY_CLOUD, ++ ZXDH_DEV_ID_ADAPTIVE_E310_TY_CLOUD_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E310_TY_CLOUD, ++ ZXDH_DEV_ID_ADAPTIVE_E310_TY_CLOUD_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312S_D, ZXDH_DEV_ID_ADAPTIVE_E312S_D_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312S_D, ZXDH_DEV_ID_ADAPTIVE_E312S_D_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312_XUANWU, ++ ZXDH_DEV_ID_ADAPTIVE_E312_XUANWU_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312_XUANWU, ++ ZXDH_DEV_ID_ADAPTIVE_E312_XUANWU_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E316_XUANWU, ++ ZXDH_DEV_ID_ADAPTIVE_E316_XUANWU_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E316_XUANWU, ++ ZXDH_DEV_ID_ADAPTIVE_E316_XUANWU_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E318_XUANWU, ++ ZXDH_DEV_ID_ADAPTIVE_E318_XUANWU_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E318_XUANWU, ++ ZXDH_DEV_ID_ADAPTIVE_E318_XUANWU_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E316_XUANWU_LX, ++ ZXDH_DEV_ID_ADAPTIVE_E316_XUANWU_LX_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E316_XUANWU_LX, ++ ZXDH_DEV_ID_ADAPTIVE_E316_XUANWU_LX_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E318_XUANWU_LX, ++ ZXDH_DEV_ID_ADAPTIVE_E318_XUANWU_LX_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E318_XUANWU_LX, ++ ZXDH_DEV_ID_ADAPTIVE_E318_XUANWU_LX_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_N25HX2F200G, ++ ZXDH_DEV_ID_ADAPTIVE_N25HX2F200G_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_N25HX2F200G, ++ ZXDH_DEV_ID_ADAPTIVE_N25HX2F200G_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_N25HX1F400G, ++ ZXDH_DEV_ID_ADAPTIVE_N25HX1F400G_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_N25HX1F400G, ++ ZXDH_DEV_ID_ADAPTIVE_N25HX1F400G_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E310_RDMA_XUANWU_YZ, ++ ZXDH_DEV_ID_ADAPTIVE_E310_RDMA_XUANWU_YZ_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E310_RDMA_XUANWU_YZ, ++ ZXDH_DEV_ID_ADAPTIVE_E310_RDMA_XUANWU_YZ_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312_RDMA_XUANWU_YZ, ++ ZXDH_DEV_ID_ADAPTIVE_E312_RDMA_XUANWU_YZ_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312_RDMA_XUANWU_YZ, ++ ZXDH_DEV_ID_ADAPTIVE_E312_RDMA_XUANWU_YZ_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312_S_XUANWU_YZ, ++ ZXDH_DEV_ID_ADAPTIVE_E312_S_XUANWU_YZ_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312_S_XUANWU_YZ, ++ ZXDH_DEV_ID_ADAPTIVE_E312_S_XUANWU_YZ_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E316_L_XUANWU_YZ, ++ ZXDH_DEV_ID_ADAPTIVE_E316_L_XUANWU_YZ_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E316_L_XUANWU_YZ, ++ ZXDH_DEV_ID_ADAPTIVE_E316_L_XUANWU_YZ_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E316_XUANWU_YZ, ++ ZXDH_DEV_ID_ADAPTIVE_E316_XUANWU_YZ_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E316_XUANWU_YZ, ++ ZXDH_DEV_ID_ADAPTIVE_E316_XUANWU_YZ_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E318_XUANWU_YZ, ++ ZXDH_DEV_ID_ADAPTIVE_E318_XUANWU_YZ_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E318_XUANWU_YZ, ++ ZXDH_DEV_ID_ADAPTIVE_E318_XUANWU_YZ_VF), ++ {} ++}; ++ ++/** ++ * zxdh_ufree_context - free context that was allocated ++ * @ibctx: context allocated ptr ++ */ ++static void zxdh_ufree_context(struct ibv_context *ibctx) ++{ ++ struct zxdh_uvcontext *iwvctx; ++ ++ iwvctx = container_of(ibctx, struct zxdh_uvcontext, ibv_ctx.context); ++ ++ zxdh_ufree_pd(&iwvctx->iwupd->ibv_pd); ++ zxdh_munmap(iwvctx->srq_db, iwvctx->max_hw_srq_mmap_size); ++ zxdh_munmap(iwvctx->sq_db, ZXDH_HW_PAGE_SIZE); ++ zxdh_munmap(iwvctx->cq_db, ZXDH_HW_PAGE_SIZE); ++ if (iwvctx->is_qp_ext_mem_support) { ++ zxdh_munmap(iwvctx->ext_qp_buff_base, ZXDH_HW_PAGE_SIZE_L2D); ++ zxdh_munmap(iwvctx->ext_sq_db, ZXDH_HW_PAGE_SIZE); ++ } ++ verbs_uninit_context(&iwvctx->ibv_ctx); ++ free(iwvctx); ++} ++ ++static const struct verbs_context_ops zxdh_uctx_ops = { ++ .alloc_mw = zxdh_ualloc_mw, ++ .alloc_pd = zxdh_ualloc_pd, ++ .attach_mcast = zxdh_uattach_mcast, ++ .bind_mw = zxdh_ubind_mw, ++ .cq_event = zxdh_cq_event, ++ .create_ah = zxdh_ucreate_ah, ++ .create_cq = zxdh_ucreate_cq, ++ .create_cq_ex = zxdh_ucreate_cq_ex, ++ .create_qp = zxdh_ucreate_qp, ++ .create_qp_ex = zxdh_ucreate_qp_ex, ++ .create_srq = zxdh_ucreate_srq, ++ .dealloc_mw = zxdh_udealloc_mw, ++ .dealloc_pd = zxdh_ufree_pd, ++ .dereg_mr = zxdh_udereg_mr, ++ .destroy_ah = zxdh_udestroy_ah, ++ .destroy_cq = zxdh_udestroy_cq, ++ .modify_cq = zxdh_umodify_cq, ++ .destroy_qp = zxdh_udestroy_qp, ++ .destroy_srq = zxdh_udestroy_srq, ++ .detach_mcast = zxdh_udetach_mcast, ++ .modify_qp = zxdh_umodify_qp, ++ .modify_srq = zxdh_umodify_srq, ++ .poll_cq = zxdh_upoll_cq, ++ .post_recv = zxdh_upost_recv, ++ .post_send = zxdh_upost_send, ++ .post_srq_recv = zxdh_upost_srq_recv, ++ .query_device_ex = zxdh_uquery_device_ex, ++#if IBVERBS_PABI_VERSION == 25 ++ .query_device = zxdh_uquery_device, ++#endif ++ .query_port = zxdh_uquery_port, ++ .query_qp = zxdh_uquery_qp, ++ .query_srq = zxdh_uquery_srq, ++ .reg_mr = zxdh_ureg_mr, ++ .rereg_mr = zxdh_urereg_mr, ++ .req_notify_cq = zxdh_uarm_cq, ++ .resize_cq = zxdh_uresize_cq, ++ .free_context = zxdh_ufree_context, ++ .get_srq_num = zxdh_uget_srq_num, ++}; ++ ++static int ibv_cmd_get_context_wrap(struct verbs_context *context_ex, ++ struct ibv_get_context *cmd, size_t cmd_size, ++ struct ib_uverbs_get_context_resp *resp, size_t resp_size) ++{ ++#if IBVERBS_PABI_VERSION >= 57 ++ return ibv_cmd_get_context(context_ex, cmd, cmd_size, NULL, resp, resp_size); ++#else ++ return ibv_cmd_get_context(context_ex, cmd, cmd_size, resp, resp_size); ++#endif ++} ++/** ++ * zxdh_ualloc_context - allocate context for user app ++ * @ibdev: ib device created during zxdh_driver_init ++ * @cmd_fd: save fd for the device ++ * @private_data: device private data ++ * ++ * Returns callback routine table and calls driver for allocating ++ * context and getting back resource information to return as ibv_context. ++ */ ++static struct verbs_context *zxdh_ualloc_context(struct ibv_device *ibdev, int cmd_fd, void *private_data) ++{ ++ struct ibv_pd *ibv_pd; ++ struct zxdh_get_context cmd; ++ struct zxdh_get_context_resp resp = {0}; ++ __u64 sq_db_mmap_key, cq_db_mmap_key, srq_db_mmap_key, ext_sq_db_mmap_key, ext_qp_buff_base_mmap_key; ++ __u32 max_hw_srq_mmap_size = ZXDH_HW_PAGE_SIZE; ++ struct zxdh_uvcontext *iwvctx = verbs_init_and_alloc_context(ibdev, cmd_fd, iwvctx, ibv_ctx, RDMA_DRIVER_ZXDH); ++ if (!iwvctx) ++ return NULL; ++ ++ zxdh_set_debug_mask(); ++ iwvctx->zxdh_write_imm_split_switch = zxdh_get_write_imm_split_switch(); ++ ++ cmd.userspace_ver = ZXDH_CONTEXT_VER_V3; ++ if (ibv_cmd_get_context_wrap(&iwvctx->ibv_ctx, (struct ibv_get_context *)&cmd, sizeof(cmd), &resp.ibv_resp, sizeof(resp))) { ++ cmd.userspace_ver = ZXDH_CONTEXT_VER_V2; ++ if (ibv_cmd_get_context_wrap(&iwvctx->ibv_ctx, (struct ibv_get_context *)&cmd, sizeof(cmd), &resp.ibv_resp, sizeof(resp))) { ++ cmd.userspace_ver = ZXDH_CONTEXT_VER_V1; ++ if (ibv_cmd_get_context_wrap(&iwvctx->ibv_ctx, (struct ibv_get_context *)&cmd, sizeof(cmd), &resp.ibv_resp, sizeof(resp))) { ++ goto err_free; ++ } ++ } ++ } ++ ++ verbs_set_ops(&iwvctx->ibv_ctx, &zxdh_uctx_ops); ++ ++ iwvctx->dev_attrs.feature_flags = resp.feature_flags; ++ iwvctx->dev_attrs.max_hw_wq_frags = resp.max_hw_wq_frags; ++ iwvctx->dev_attrs.max_hw_read_sges = resp.max_hw_read_sges; ++ iwvctx->dev_attrs.max_hw_inline = resp.max_hw_inline; ++ iwvctx->dev_attrs.max_hw_rq_quanta = resp.max_hw_rq_quanta; ++ iwvctx->dev_attrs.max_hw_srq_quanta = resp.max_hw_srq_quanta; ++ iwvctx->dev_attrs.max_hw_wq_quanta = resp.max_hw_wq_quanta; ++ iwvctx->dev_attrs.max_hw_srq_wr = resp.max_hw_srq_wr; ++ iwvctx->dev_attrs.max_hw_sq_chunk = resp.max_hw_sq_chunk; ++ iwvctx->dev_attrs.max_hw_cq_size = resp.max_hw_cq_size; ++ iwvctx->dev_attrs.min_hw_cq_size = resp.min_hw_cq_size; ++ iwvctx->abi_ver = ZXDH_ABI_VER; ++ iwvctx->dev_attrs.chip_rev = resp.chip_rev; ++ iwvctx->dev_attrs.rdma_tool_flags = resp.rdma_tool_flags; ++ if (iwvctx->dev_attrs.rdma_tool_flags == 0) { ++ iwvctx->dev_attrs.rdma_tool_flags = ++ ZXDH_QP_EXTEND_OP | ZXDH_CAPTURE | ZXDH_GET_HW_DATA; ++ } ++ ++ sq_db_mmap_key = resp.sq_db_mmap_key; ++ cq_db_mmap_key = resp.cq_db_mmap_key; ++ ext_sq_db_mmap_key = resp.ext_sq_db_mmap_key; ++ ext_qp_buff_base_mmap_key = resp.l2d_addr_mmap_key; ++ iwvctx->is_qp_ext_mem_support = resp.is_qp_ext_mem_support; ++ iwvctx->dev_attrs.max_hw_rq_sges = resp.max_hw_rq_sges; ++ iwvctx->dev_attrs.db_addr_type = resp.db_addr_type; ++ iwvctx->dev_attrs.sq_db_pa = resp.sq_db_pa; ++ iwvctx->dev_attrs.cq_db_pa = resp.cq_db_pa; ++ if (ZXDH_SYS_PAGE_SIZE == ZXDH_HW_64K_PAGE_SIZE) { ++ iwvctx->dev_attrs.sq_db_bar_off = resp.sq_db_bar_off; ++ iwvctx->dev_attrs.cq_db_bar_off = resp.cq_db_bar_off; ++ } ++ iwvctx->dev_attrs.srq_db_bar_off = resp.srq_db_bar_off; ++ ++ if (iwvctx->dev_attrs.db_addr_type != ZXDH_DB_ADDR_BAR) ++ goto err_free; ++ if (cmd.userspace_ver >= ZXDH_CONTEXT_VER_V3) { ++ iwvctx->max_hw_srq_mmap_size = resp.srq_db_mmap_size; ++ max_hw_srq_mmap_size = resp.srq_db_mmap_size; ++ } ++ if (cmd.userspace_ver >= ZXDH_CONTEXT_VER_V2) { ++ srq_db_mmap_key = resp.srq_db_mmap_key; ++ if (srq_db_mmap_key) { ++ iwvctx->srq_db = zxdh_mmap(cmd_fd, max_hw_srq_mmap_size, srq_db_mmap_key); ++ if (iwvctx->srq_db == MAP_FAILED) ++ goto err_free; ++ } ++ } ++ ++ iwvctx->sq_db = zxdh_mmap(cmd_fd, ZXDH_HW_PAGE_SIZE, sq_db_mmap_key); ++ if (iwvctx->sq_db == MAP_FAILED) { ++ zxdh_munmap(iwvctx->srq_db, max_hw_srq_mmap_size); ++ goto err_free; ++ } ++ iwvctx->cq_db = zxdh_mmap(cmd_fd, ZXDH_HW_PAGE_SIZE, cq_db_mmap_key); ++ if (iwvctx->cq_db == MAP_FAILED) { ++ zxdh_munmap(iwvctx->srq_db, max_hw_srq_mmap_size); ++ zxdh_munmap(iwvctx->sq_db, ZXDH_HW_PAGE_SIZE); ++ goto err_free; ++ } ++ ++ iwvctx->ext_qp_buff_base = NULL; ++ iwvctx->ext_sq_db = NULL; ++ if (iwvctx->is_qp_ext_mem_support) { ++ iwvctx->ext_sq_db = zxdh_mmap(cmd_fd, ZXDH_HW_PAGE_SIZE, ext_sq_db_mmap_key); ++ if (iwvctx->ext_sq_db == MAP_FAILED) { ++ zxdh_munmap(iwvctx->srq_db, max_hw_srq_mmap_size); ++ zxdh_munmap(iwvctx->sq_db, ZXDH_HW_PAGE_SIZE); ++ zxdh_munmap(iwvctx->cq_db, ZXDH_HW_PAGE_SIZE); ++ goto err_free; ++ } ++ iwvctx->ext_qp_buff_base = zxdh_mmap(cmd_fd, ZXDH_HW_PAGE_SIZE_L2D, ext_qp_buff_base_mmap_key); ++ if (iwvctx->ext_qp_buff_base == MAP_FAILED) { ++ zxdh_munmap(iwvctx->srq_db, max_hw_srq_mmap_size); ++ zxdh_munmap(iwvctx->sq_db, ZXDH_HW_PAGE_SIZE); ++ zxdh_munmap(iwvctx->cq_db, ZXDH_HW_PAGE_SIZE); ++ zxdh_munmap(iwvctx->ext_sq_db, ZXDH_HW_PAGE_SIZE); ++ goto err_free; ++ } ++ } ++ ++ ibv_pd = zxdh_ualloc_pd(&iwvctx->ibv_ctx.context); ++ if (!ibv_pd) { ++ zxdh_munmap(iwvctx->srq_db, max_hw_srq_mmap_size); ++ zxdh_munmap(iwvctx->sq_db, ZXDH_HW_PAGE_SIZE); ++ zxdh_munmap(iwvctx->cq_db, ZXDH_HW_PAGE_SIZE); ++ if (iwvctx->is_qp_ext_mem_support) { ++ zxdh_munmap(iwvctx->ext_sq_db, ZXDH_HW_PAGE_SIZE); ++ zxdh_munmap(iwvctx->ext_qp_buff_base, ZXDH_HW_PAGE_SIZE_L2D); ++ } ++ goto err_free; ++ } ++ ++ ibv_pd->context = &iwvctx->ibv_ctx.context; ++ iwvctx->iwupd = container_of(ibv_pd, struct zxdh_upd, ibv_pd); ++ add_private_ops(iwvctx); ++ return &iwvctx->ibv_ctx; ++err_free: ++ free(iwvctx); ++ return NULL; ++} ++ ++static void zxdh_uninit_device(struct verbs_device *verbs_device) ++{ ++ struct zxdh_udevice *dev; ++ ++ dev = container_of(&verbs_device->device, struct zxdh_udevice, ++ ibv_dev.device); ++ free(dev); ++} ++ ++static struct verbs_device *zxdh_device_alloc(struct verbs_sysfs_dev *sysfs_dev) ++{ ++ struct zxdh_udevice *dev; ++ ++ dev = calloc(1, sizeof(*dev)); ++ if (!dev) ++ return NULL; ++ ++ return &dev->ibv_dev; ++} ++ ++static const struct verbs_device_ops zxdh_udev_ops = { ++ .alloc_context = zxdh_ualloc_context, ++ .alloc_device = zxdh_device_alloc, ++ .match_max_abi_version = ZXDH_MAX_ABI_VERSION, ++ .match_min_abi_version = ZXDH_MIN_ABI_VERSION, ++ .match_table = hca_table, ++ .name = "zxdh", ++ .uninit_device = zxdh_uninit_device, ++}; ++PROVIDER_DRIVER(zxdh, zxdh_udev_ops); +diff -ruN baseline/providers/zrdma/zxdh_zrdma.c.tmp b/providers/zrdma/zxdh_zrdma.c.tmp +--- a/providers/zrdma/zxdh_zrdma.c.tmp 1970-01-01 08:00:00.000000000 +0800 ++++ b/providers/zrdma/zxdh_zrdma.c.tmp 2026-08-06 15:29:37.736709298 +0800 +@@ -0,0 +1,345 @@ ++// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB ++/* Copyright (c) 2024 ZTE Corporation. All rights reserved. */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "zxdh_devids.h" ++#include "zxdh_zrdma.h" ++#include "zxdh_abi.h" ++#include "private_verbs_cmd.h" ++ ++ ++#define ZXDH_HCA(v, d) VERBS_PCI_MATCH(v, d, NULL) ++static const struct verbs_match_ent hca_table[] = { ++ VERBS_DRIVER_ID(RDMA_DRIVER_ZXDH), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_EVB, ZXDH_DEV_ID_ADAPTIVE_EVB_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_EVB, ZXDH_DEV_ID_ADAPTIVE_EVB_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312, ZXDH_DEV_ID_ADAPTIVE_E312_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312, ZXDH_DEV_ID_ADAPTIVE_E312_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E310, ZXDH_DEV_ID_ADAPTIVE_E310_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E310, ZXDH_DEV_ID_ADAPTIVE_E310_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E310_RDMA, ++ ZXDH_DEV_ID_ADAPTIVE_E310_RDMA_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E310_RDMA, ++ ZXDH_DEV_ID_ADAPTIVE_E310_RDMA_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E316, ZXDH_DEV_ID_ADAPTIVE_E316_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E316, ZXDH_DEV_ID_ADAPTIVE_E316_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E318, ZXDH_DEV_ID_ADAPTIVE_E318_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E318, ZXDH_DEV_ID_ADAPTIVE_E318_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_X512, ZXDH_DEV_ID_ADAPTIVE_X512_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_X512, ZXDH_DEV_ID_ADAPTIVE_X512_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312_TY_CLOUD, ++ ZXDH_DEV_ID_ADAPTIVE_E312_TY_CLOUD_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312_TY_CLOUD, ++ ZXDH_DEV_ID_ADAPTIVE_E312_TY_CLOUD_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E310_TY_CLOUD, ++ ZXDH_DEV_ID_ADAPTIVE_E310_TY_CLOUD_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E310_TY_CLOUD, ++ ZXDH_DEV_ID_ADAPTIVE_E310_TY_CLOUD_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312S_D, ZXDH_DEV_ID_ADAPTIVE_E312S_D_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312S_D, ZXDH_DEV_ID_ADAPTIVE_E312S_D_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312_XUANWU, ++ ZXDH_DEV_ID_ADAPTIVE_E312_XUANWU_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312_XUANWU, ++ ZXDH_DEV_ID_ADAPTIVE_E312_XUANWU_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E316_XUANWU, ++ ZXDH_DEV_ID_ADAPTIVE_E316_XUANWU_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E316_XUANWU, ++ ZXDH_DEV_ID_ADAPTIVE_E316_XUANWU_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E318_XUANWU, ++ ZXDH_DEV_ID_ADAPTIVE_E318_XUANWU_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E318_XUANWU, ++ ZXDH_DEV_ID_ADAPTIVE_E318_XUANWU_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E316_XUANWU_LX, ++ ZXDH_DEV_ID_ADAPTIVE_E316_XUANWU_LX_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E316_XUANWU_LX, ++ ZXDH_DEV_ID_ADAPTIVE_E316_XUANWU_LX_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E318_XUANWU_LX, ++ ZXDH_DEV_ID_ADAPTIVE_E318_XUANWU_LX_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E318_XUANWU_LX, ++ ZXDH_DEV_ID_ADAPTIVE_E318_XUANWU_LX_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_N25HX2F200G, ++ ZXDH_DEV_ID_ADAPTIVE_N25HX2F200G_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_N25HX2F200G, ++ ZXDH_DEV_ID_ADAPTIVE_N25HX2F200G_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_N25HX1F400G, ++ ZXDH_DEV_ID_ADAPTIVE_N25HX1F400G_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_N25HX1F400G, ++ ZXDH_DEV_ID_ADAPTIVE_N25HX1F400G_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E310_RDMA_XUANWU_YZ, ++ ZXDH_DEV_ID_ADAPTIVE_E310_RDMA_XUANWU_YZ_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E310_RDMA_XUANWU_YZ, ++ ZXDH_DEV_ID_ADAPTIVE_E310_RDMA_XUANWU_YZ_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312_RDMA_XUANWU_YZ, ++ ZXDH_DEV_ID_ADAPTIVE_E312_RDMA_XUANWU_YZ_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312_RDMA_XUANWU_YZ, ++ ZXDH_DEV_ID_ADAPTIVE_E312_RDMA_XUANWU_YZ_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312_S_XUANWU_YZ, ++ ZXDH_DEV_ID_ADAPTIVE_E312_S_XUANWU_YZ_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E312_S_XUANWU_YZ, ++ ZXDH_DEV_ID_ADAPTIVE_E312_S_XUANWU_YZ_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E316_L_XUANWU_YZ, ++ ZXDH_DEV_ID_ADAPTIVE_E316_L_XUANWU_YZ_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E316_L_XUANWU_YZ, ++ ZXDH_DEV_ID_ADAPTIVE_E316_L_XUANWU_YZ_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E316_XUANWU_YZ, ++ ZXDH_DEV_ID_ADAPTIVE_E316_XUANWU_YZ_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E316_XUANWU_YZ, ++ ZXDH_DEV_ID_ADAPTIVE_E316_XUANWU_YZ_VF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E318_XUANWU_YZ, ++ ZXDH_DEV_ID_ADAPTIVE_E318_XUANWU_YZ_PF), ++ ZXDH_HCA(PCI_VENDOR_ID_ZXDH_E318_XUANWU_YZ, ++ ZXDH_DEV_ID_ADAPTIVE_E318_XUANWU_YZ_VF), ++ {} ++}; ++ ++/** ++ * zxdh_ufree_context - free context that was allocated ++ * @ibctx: context allocated ptr ++ */ ++static void zxdh_ufree_context(struct ibv_context *ibctx) ++{ ++ struct zxdh_uvcontext *iwvctx; ++ ++ iwvctx = container_of(ibctx, struct zxdh_uvcontext, ibv_ctx.context); ++ ++ zxdh_ufree_pd(&iwvctx->iwupd->ibv_pd); ++ zxdh_munmap(iwvctx->srq_db, iwvctx->max_hw_srq_mmap_size); ++ zxdh_munmap(iwvctx->sq_db, ZXDH_HW_PAGE_SIZE); ++ zxdh_munmap(iwvctx->cq_db, ZXDH_HW_PAGE_SIZE); ++ if (iwvctx->is_qp_ext_mem_support) { ++ zxdh_munmap(iwvctx->ext_qp_buff_base, ZXDH_HW_PAGE_SIZE_L2D); ++ zxdh_munmap(iwvctx->ext_sq_db, ZXDH_HW_PAGE_SIZE); ++ } ++ verbs_uninit_context(&iwvctx->ibv_ctx); ++ free(iwvctx); ++} ++ ++static const struct verbs_context_ops zxdh_uctx_ops = { ++ .alloc_mw = zxdh_ualloc_mw, ++ .alloc_pd = zxdh_ualloc_pd, ++ .attach_mcast = zxdh_uattach_mcast, ++ .bind_mw = zxdh_ubind_mw, ++ .cq_event = zxdh_cq_event, ++ .create_ah = zxdh_ucreate_ah, ++ .create_cq = zxdh_ucreate_cq, ++ .create_cq_ex = zxdh_ucreate_cq_ex, ++ .create_qp = zxdh_ucreate_qp, ++ .create_qp_ex = zxdh_ucreate_qp_ex, ++ .create_srq = zxdh_ucreate_srq, ++ .dealloc_mw = zxdh_udealloc_mw, ++ .dealloc_pd = zxdh_ufree_pd, ++ .dereg_mr = zxdh_udereg_mr, ++ .destroy_ah = zxdh_udestroy_ah, ++ .destroy_cq = zxdh_udestroy_cq, ++ .modify_cq = zxdh_umodify_cq, ++ .destroy_qp = zxdh_udestroy_qp, ++ .destroy_srq = zxdh_udestroy_srq, ++ .detach_mcast = zxdh_udetach_mcast, ++ .modify_qp = zxdh_umodify_qp, ++ .modify_srq = zxdh_umodify_srq, ++ .poll_cq = zxdh_upoll_cq, ++ .post_recv = zxdh_upost_recv, ++ .post_send = zxdh_upost_send, ++ .post_srq_recv = zxdh_upost_srq_recv, ++ .query_device_ex = zxdh_uquery_device_ex, ++#if IBVERBS_PABI_VERSION == 25 ++ .query_device = zxdh_uquery_device, ++#endif ++ .query_port = zxdh_uquery_port, ++ .query_qp = zxdh_uquery_qp, ++ .query_srq = zxdh_uquery_srq, ++ .reg_dmabuf_mr = zxdh_ureg_mr_dmabuf, ++ .reg_mr = zxdh_ureg_mr, ++ .rereg_mr = zxdh_urereg_mr, ++ .req_notify_cq = zxdh_uarm_cq, ++ .resize_cq = zxdh_uresize_cq, ++ .free_context = zxdh_ufree_context, ++ .get_srq_num = zxdh_uget_srq_num, ++}; ++ ++static int ibv_cmd_get_context_wrap(struct verbs_context *context_ex, ++ struct ibv_get_context *cmd, size_t cmd_size, ++ struct ib_uverbs_get_context_resp *resp, size_t resp_size) ++{ ++#if IBVERBS_PABI_VERSION >= 57 ++ return ibv_cmd_get_context(context_ex, cmd, cmd_size, NULL, resp, resp_size); ++#else ++ return ibv_cmd_get_context(context_ex, cmd, cmd_size, resp, resp_size); ++#endif ++} ++/** ++ * zxdh_ualloc_context - allocate context for user app ++ * @ibdev: ib device created during zxdh_driver_init ++ * @cmd_fd: save fd for the device ++ * @private_data: device private data ++ * ++ * Returns callback routine table and calls driver for allocating ++ * context and getting back resource information to return as ibv_context. ++ */ ++static struct verbs_context *zxdh_ualloc_context(struct ibv_device *ibdev, int cmd_fd, void *private_data) ++{ ++ struct ibv_pd *ibv_pd; ++ struct zxdh_get_context cmd; ++ struct zxdh_get_context_resp resp = {0}; ++ __u64 sq_db_mmap_key, cq_db_mmap_key, srq_db_mmap_key, ext_sq_db_mmap_key, ext_qp_buff_base_mmap_key; ++ __u32 max_hw_srq_mmap_size = ZXDH_HW_PAGE_SIZE; ++ struct zxdh_uvcontext *iwvctx = verbs_init_and_alloc_context(ibdev, cmd_fd, iwvctx, ibv_ctx, RDMA_DRIVER_ZXDH); ++ if (!iwvctx) ++ return NULL; ++ ++ zxdh_set_debug_mask(); ++ iwvctx->zxdh_write_imm_split_switch = zxdh_get_write_imm_split_switch(); ++ ++ cmd.userspace_ver = ZXDH_CONTEXT_VER_V3; ++ if (ibv_cmd_get_context_wrap(&iwvctx->ibv_ctx, (struct ibv_get_context *)&cmd, sizeof(cmd), &resp.ibv_resp, sizeof(resp))) { ++ cmd.userspace_ver = ZXDH_CONTEXT_VER_V2; ++ if (ibv_cmd_get_context_wrap(&iwvctx->ibv_ctx, (struct ibv_get_context *)&cmd, sizeof(cmd), &resp.ibv_resp, sizeof(resp))) { ++ cmd.userspace_ver = ZXDH_CONTEXT_VER_V1; ++ if (ibv_cmd_get_context_wrap(&iwvctx->ibv_ctx, (struct ibv_get_context *)&cmd, sizeof(cmd), &resp.ibv_resp, sizeof(resp))) { ++ goto err_free; ++ } ++ } ++ } ++ ++ verbs_set_ops(&iwvctx->ibv_ctx, &zxdh_uctx_ops); ++ ++ iwvctx->dev_attrs.feature_flags = resp.feature_flags; ++ iwvctx->dev_attrs.max_hw_wq_frags = resp.max_hw_wq_frags; ++ iwvctx->dev_attrs.max_hw_read_sges = resp.max_hw_read_sges; ++ iwvctx->dev_attrs.max_hw_inline = resp.max_hw_inline; ++ iwvctx->dev_attrs.max_hw_rq_quanta = resp.max_hw_rq_quanta; ++ iwvctx->dev_attrs.max_hw_srq_quanta = resp.max_hw_srq_quanta; ++ iwvctx->dev_attrs.max_hw_wq_quanta = resp.max_hw_wq_quanta; ++ iwvctx->dev_attrs.max_hw_srq_wr = resp.max_hw_srq_wr; ++ iwvctx->dev_attrs.max_hw_sq_chunk = resp.max_hw_sq_chunk; ++ iwvctx->dev_attrs.max_hw_cq_size = resp.max_hw_cq_size; ++ iwvctx->dev_attrs.min_hw_cq_size = resp.min_hw_cq_size; ++ iwvctx->abi_ver = ZXDH_ABI_VER; ++ iwvctx->dev_attrs.chip_rev = resp.chip_rev; ++ iwvctx->dev_attrs.rdma_tool_flags = resp.rdma_tool_flags; ++ if (iwvctx->dev_attrs.rdma_tool_flags == 0) { ++ iwvctx->dev_attrs.rdma_tool_flags = ++ ZXDH_QP_EXTEND_OP | ZXDH_CAPTURE | ZXDH_GET_HW_DATA; ++ } ++ ++ sq_db_mmap_key = resp.sq_db_mmap_key; ++ cq_db_mmap_key = resp.cq_db_mmap_key; ++ ext_sq_db_mmap_key = resp.ext_sq_db_mmap_key; ++ ext_qp_buff_base_mmap_key = resp.l2d_addr_mmap_key; ++ iwvctx->is_qp_ext_mem_support = resp.is_qp_ext_mem_support; ++ iwvctx->dev_attrs.max_hw_rq_sges = resp.max_hw_rq_sges; ++ iwvctx->dev_attrs.db_addr_type = resp.db_addr_type; ++ iwvctx->dev_attrs.sq_db_pa = resp.sq_db_pa; ++ iwvctx->dev_attrs.cq_db_pa = resp.cq_db_pa; ++ if (ZXDH_SYS_PAGE_SIZE == ZXDH_HW_64K_PAGE_SIZE) { ++ iwvctx->dev_attrs.sq_db_bar_off = resp.sq_db_bar_off; ++ iwvctx->dev_attrs.cq_db_bar_off = resp.cq_db_bar_off; ++ } ++ iwvctx->dev_attrs.srq_db_bar_off = resp.srq_db_bar_off; ++ ++ if (iwvctx->dev_attrs.db_addr_type != ZXDH_DB_ADDR_BAR) ++ goto err_free; ++ if (cmd.userspace_ver >= ZXDH_CONTEXT_VER_V3) { ++ iwvctx->max_hw_srq_mmap_size = resp.srq_db_mmap_size; ++ max_hw_srq_mmap_size = resp.srq_db_mmap_size; ++ } ++ if (cmd.userspace_ver >= ZXDH_CONTEXT_VER_V2) { ++ srq_db_mmap_key = resp.srq_db_mmap_key; ++ if (srq_db_mmap_key) { ++ iwvctx->srq_db = zxdh_mmap(cmd_fd, max_hw_srq_mmap_size, srq_db_mmap_key); ++ if (iwvctx->srq_db == MAP_FAILED) ++ goto err_free; ++ } ++ } ++ ++ iwvctx->sq_db = zxdh_mmap(cmd_fd, ZXDH_HW_PAGE_SIZE, sq_db_mmap_key); ++ if (iwvctx->sq_db == MAP_FAILED) { ++ zxdh_munmap(iwvctx->srq_db, max_hw_srq_mmap_size); ++ goto err_free; ++ } ++ iwvctx->cq_db = zxdh_mmap(cmd_fd, ZXDH_HW_PAGE_SIZE, cq_db_mmap_key); ++ if (iwvctx->cq_db == MAP_FAILED) { ++ zxdh_munmap(iwvctx->srq_db, max_hw_srq_mmap_size); ++ zxdh_munmap(iwvctx->sq_db, ZXDH_HW_PAGE_SIZE); ++ goto err_free; ++ } ++ ++ iwvctx->ext_qp_buff_base = NULL; ++ iwvctx->ext_sq_db = NULL; ++ if (iwvctx->is_qp_ext_mem_support) { ++ iwvctx->ext_sq_db = zxdh_mmap(cmd_fd, ZXDH_HW_PAGE_SIZE, ext_sq_db_mmap_key); ++ if (iwvctx->ext_sq_db == MAP_FAILED) { ++ zxdh_munmap(iwvctx->srq_db, max_hw_srq_mmap_size); ++ zxdh_munmap(iwvctx->sq_db, ZXDH_HW_PAGE_SIZE); ++ zxdh_munmap(iwvctx->cq_db, ZXDH_HW_PAGE_SIZE); ++ goto err_free; ++ } ++ iwvctx->ext_qp_buff_base = zxdh_mmap(cmd_fd, ZXDH_HW_PAGE_SIZE_L2D, ext_qp_buff_base_mmap_key); ++ if (iwvctx->ext_qp_buff_base == MAP_FAILED) { ++ zxdh_munmap(iwvctx->srq_db, max_hw_srq_mmap_size); ++ zxdh_munmap(iwvctx->sq_db, ZXDH_HW_PAGE_SIZE); ++ zxdh_munmap(iwvctx->cq_db, ZXDH_HW_PAGE_SIZE); ++ zxdh_munmap(iwvctx->ext_sq_db, ZXDH_HW_PAGE_SIZE); ++ goto err_free; ++ } ++ } ++ ++ ibv_pd = zxdh_ualloc_pd(&iwvctx->ibv_ctx.context); ++ if (!ibv_pd) { ++ zxdh_munmap(iwvctx->srq_db, max_hw_srq_mmap_size); ++ zxdh_munmap(iwvctx->sq_db, ZXDH_HW_PAGE_SIZE); ++ zxdh_munmap(iwvctx->cq_db, ZXDH_HW_PAGE_SIZE); ++ if (iwvctx->is_qp_ext_mem_support) { ++ zxdh_munmap(iwvctx->ext_sq_db, ZXDH_HW_PAGE_SIZE); ++ zxdh_munmap(iwvctx->ext_qp_buff_base, ZXDH_HW_PAGE_SIZE_L2D); ++ } ++ goto err_free; ++ } ++ ++ ibv_pd->context = &iwvctx->ibv_ctx.context; ++ iwvctx->iwupd = container_of(ibv_pd, struct zxdh_upd, ibv_pd); ++ add_private_ops(iwvctx); ++ return &iwvctx->ibv_ctx; ++err_free: ++ free(iwvctx); ++ return NULL; ++} ++ ++static void zxdh_uninit_device(struct verbs_device *verbs_device) ++{ ++ struct zxdh_udevice *dev; ++ ++ dev = container_of(&verbs_device->device, struct zxdh_udevice, ++ ibv_dev.device); ++ free(dev); ++} ++ ++static struct verbs_device *zxdh_device_alloc(struct verbs_sysfs_dev *sysfs_dev) ++{ ++ struct zxdh_udevice *dev; ++ ++ dev = calloc(1, sizeof(*dev)); ++ if (!dev) ++ return NULL; ++ ++ return &dev->ibv_dev; ++} ++ ++static const struct verbs_device_ops zxdh_udev_ops = { ++ .alloc_context = zxdh_ualloc_context, ++ .alloc_device = zxdh_device_alloc, ++ .match_max_abi_version = ZXDH_MAX_ABI_VERSION, ++ .match_min_abi_version = ZXDH_MIN_ABI_VERSION, ++ .match_table = hca_table, ++ .name = "zxdh", ++ .uninit_device = zxdh_uninit_device, ++}; ++PROVIDER_DRIVER(zxdh, zxdh_udev_ops); +diff -ruN baseline/providers/zrdma/zxdh_zrdma.h b/providers/zrdma/zxdh_zrdma.h +--- a/providers/zrdma/zxdh_zrdma.h 1970-01-01 08:00:00.000000000 +0800 ++++ b/providers/zrdma/zxdh_zrdma.h 2026-08-06 15:29:36.798644877 +0800 +@@ -0,0 +1,302 @@ ++/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */ ++/* Copyright (c) 2024 ZTE Corporation. All rights reserved. */ ++#ifndef ZXDH_UMAIN_H ++#define ZXDH_UMAIN_H ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "zxdh_status.h" ++#include "zxdh_verbs.h" ++ ++#define ZXDH_BASE_PUSH_PAGE 1 ++#define ZXDH_U_MINCQ_SIZE 256 ++#define ZXDH_DB_SHADOW_AREA_SIZE 8 ++#define ZXDH_DB_SQ_OFFSET 0x404 ++#define ZXDH_DB_CQ_OFFSET 0x588 ++ ++#define MIN_UDP_SPORT 1024 ++#define MIN_QP_QPN 1 ++ ++enum zxdh_supported_wc_flags { ++ ZXDH_CQ_SUPPORTED_WC_FLAGS = ++ IBV_WC_EX_WITH_BYTE_LEN | IBV_WC_EX_WITH_IMM | ++ IBV_WC_EX_WITH_QP_NUM | IBV_WC_EX_WITH_SRC_QP | ++ IBV_WC_EX_WITH_SLID | IBV_WC_EX_WITH_SL | ++ IBV_WC_EX_WITH_DLID_PATH_BITS | ++ IBV_WC_EX_WITH_COMPLETION_TIMESTAMP_WALLCLOCK | ++ IBV_WC_EX_WITH_COMPLETION_TIMESTAMP, ++}; ++ ++enum { ++ ZXDH_DBG_QP = 1 << 0, ++ ZXDH_DBG_CQ = 1 << 1, ++ ZXDH_DBG_SRQ = 1 << 2, ++ ZXDH_DBG_QUANTA = 1 << 3, ++}; ++extern uint32_t zxdh_debug_mask; ++#define zxdh_dbg(mask, format, arg...) \ ++ do { \ ++ if (mask & zxdh_debug_mask) { \ ++ int tmp = errno; \ ++ (void)fprintf(stdout, "%s:%d: " format, __func__, __LINE__, \ ++ ##arg); \ ++ errno = tmp; \ ++ } \ ++ } while (0) ++ ++#define zxdh_quanta_dbg(format, arg...) \ ++ do { \ ++ if (ZXDH_DBG_QUANTA & zxdh_debug_mask) { \ ++ int tmp = errno; \ ++ (void)fprintf(stderr, "[QUANTA_DEBUG] %s:%d: " format, \ ++ __func__, __LINE__, ##arg); \ ++ errno = tmp; \ ++ } \ ++ } while (0) ++ ++#define zxdh_status_to_errno_log(zxdh_status, format, arg...) \ ++ do { \ ++ errno = zxdh_status_code_to_errno(zxdh_status); \ ++ int tmp = errno; \ ++ (void)fprintf(stdout, "%s:%d: " "[ERROR] status=%s errno=%d: " format"\n", __func__, \ ++ __LINE__, zxdh_status_code_to_string(zxdh_status), errno, ##arg); \ ++ errno = tmp; \ ++ } while (0) ++ ++struct zxdh_udevice { ++ struct verbs_device ibv_dev; ++}; ++ ++struct zxdh_uah { ++ struct ibv_ah ibv_ah; ++ uint32_t ah_id; ++ struct ibv_global_route grh; ++}; ++ ++struct zxdh_upd { ++ struct ibv_pd ibv_pd; ++ void *arm_cq_page; ++ void *arm_cq; ++ uint32_t pd_id; ++}; ++ ++struct zxdh_uvcontext { ++ struct verbs_context ibv_ctx; ++ struct zxdh_upd *iwupd; ++ struct zxdh_dev_attrs dev_attrs; ++ void *db; ++ void *sq_db; ++ void *cq_db; ++ int abi_ver; ++ bool legacy_mode; ++ uint8_t zxdh_write_imm_split_switch; ++ struct zxdh_uvcontext_ops *cxt_ops; ++ void *srq_db; ++ uint32_t max_hw_srq_mmap_size; ++ void *ext_qp_buff_base; ++ void *ext_sq_db; ++ uint8_t is_qp_ext_mem_support; ++}; ++ ++ ++ ++struct zxdh_uqp; ++ ++struct zxdh_cq_buf { ++ struct list_node list; ++ struct zxdh_cq cq; ++ struct verbs_mr vmr; ++}; ++ ++ ++struct zxdh_ucq { ++ struct verbs_cq verbs_cq; ++ struct verbs_mr vmr; ++ struct verbs_mr vmr_shadow_area; ++ pthread_spinlock_t lock; ++ size_t buf_size; ++ bool is_armed; ++ enum zxdh_cmpl_notify last_notify; ++ int comp_vector; ++ uint32_t report_rtt; ++ struct zxdh_uqp *uqp; ++ struct zxdh_cq cq; ++ struct list_head resize_list; ++ /* for extended CQ completion fields */ ++ struct zxdh_cq_poll_info cur_cqe; ++ bool resize_enable; ++ bool is_ext_buf_mode; ++}; ++ ++struct zxdh_usrq { ++ struct ibv_srq ibv_srq; ++ struct verbs_mr vmr; ++ struct verbs_mr list_vmr; ++ struct verbs_mr db_vmr; ++ size_t total_buf_size; ++ size_t buf_size; ++ size_t list_buf_size; ++ size_t db_buf_size; ++ size_t srq_size; ++ size_t srq_list_size; ++ uint32_t srq_id; ++ uint32_t max_wr; ++ uint32_t max_sge; ++ uint32_t srq_limit; ++ pthread_spinlock_t lock; ++ uint32_t wq_size; ++ struct ibv_recv_wr *pend_rx_wr; ++ struct zxdh_srq srq; ++}; ++ ++typedef struct wqe_rate_limit_info { ++ unsigned long long gap_deadline; ++ unsigned int *vhca_wqe_db; ++ int vhca_qp_num; ++ int vhca_rate; ++ unsigned int total_wqe_size; ++ unsigned int iwuqp_qpn; ++ float param1; ++ float param2; ++ int wqe_rate_limit; ++ uint8_t first_wqe_flag; ++ uint8_t current_log2_size; ++ struct wqe_rate_limit_info *next; ++} wqe_rate_limit_info_t; ++ ++struct zxdh_uqp { ++ struct verbs_qp vqp; ++ struct zxdh_ucq *send_cq; ++ struct zxdh_ucq *recv_cq; ++ struct zxdh_usrq *srq; ++ struct verbs_mr vmr; ++ size_t buf_size; ++ uint32_t zxdh_drv_opt; ++ pthread_spinlock_t lock; ++ pthread_spinlock_t quanta_lock; ++ uint16_t sq_sig_all; ++ uint16_t qperr; ++ uint16_t rsvd; ++ uint32_t pending_rcvs; ++ uint32_t wq_size; ++ struct ibv_recv_wr *pend_rx_wr; ++ struct zxdh_qp qp; ++ enum ibv_qp_type qp_type; ++ struct zxdh_sge *recv_sges; ++ unsigned int is_srq:1; ++ unsigned int ext_buf_mode:1; ++ uint8_t inline_data[ZXDH_MAX_INLINE_DATA_SIZE]; ++ wqe_rate_limit_info_t *wqe_rate_limit_info; ++}; ++ ++struct zxdh_umr { ++ struct verbs_mr vmr; ++ uint32_t acc_flags; ++ uint8_t leaf_pbl_size; ++ uint8_t host_page_size; ++ uint64_t mr_pa_pble_index; ++}; ++ ++/* zxdh_verbs.c */ ++int zxdh_uquery_device_ex(struct ibv_context *context, ++ const struct ibv_query_device_ex_input *input, ++ struct ibv_device_attr_ex *attr, size_t attr_size); ++int zxdh_uquery_port(struct ibv_context *context, uint8_t port, ++ struct ibv_port_attr *attr); ++struct ibv_pd *zxdh_ualloc_pd(struct ibv_context *context); ++int zxdh_ufree_pd(struct ibv_pd *pd); ++struct ibv_mr *zxdh_ureg_mr(struct ibv_pd *pd, void *addr, size_t length, ++ uint64_t hca_va, int access); ++ ++ ++int zxdh_udereg_mr(struct verbs_mr *vmr); ++int zxdh_urereg_mr(struct verbs_mr *vmr, int flags, struct ibv_pd *pd, ++ void *addr, size_t length, int access); ++ ++struct ibv_mw *zxdh_ualloc_mw(struct ibv_pd *pd, enum ibv_mw_type type); ++int zxdh_ubind_mw(struct ibv_qp *qp, struct ibv_mw *mw, ++ struct ibv_mw_bind *mw_bind); ++int zxdh_udealloc_mw(struct ibv_mw *mw); ++struct ibv_cq *zxdh_ucreate_cq(struct ibv_context *context, int cqe, ++ struct ibv_comp_channel *channel, ++ int comp_vector); ++struct ibv_cq_ex *zxdh_ucreate_cq_ex(struct ibv_context *context, ++ struct ibv_cq_init_attr_ex *attr_ex); ++void zxdh_ibvcq_ex_fill_priv_funcs(struct zxdh_ucq *iwucq, ++ struct ibv_cq_init_attr_ex *attr_ex); ++int zxdh_uresize_cq(struct ibv_cq *cq, int cqe); ++int zxdh_udestroy_cq(struct ibv_cq *cq); ++int zxdh_umodify_cq(struct ibv_cq *cq, struct ibv_modify_cq_attr *attr); ++int zxdh_upoll_cq(struct ibv_cq *cq, int entries, struct ibv_wc *entry); ++int zxdh_uarm_cq(struct ibv_cq *cq, int solicited); ++void zxdh_cq_event(struct ibv_cq *cq); ++struct ibv_qp *zxdh_ucreate_qp(struct ibv_pd *pd, ++ struct ibv_qp_init_attr *attr); ++struct ibv_qp *zxdh_ucreate_qp_ex(struct ibv_context *context, ++ struct ibv_qp_init_attr_ex *attr); ++int zxdh_uquery_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask, ++ struct ibv_qp_init_attr *init_attr); ++int zxdh_umodify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask); ++int zxdh_udestroy_qp(struct ibv_qp *qp); ++int zxdh_upost_send(struct ibv_qp *ib_qp, struct ibv_send_wr *ib_wr, ++ struct ibv_send_wr **bad_wr); ++int zxdh_upost_recv(struct ibv_qp *ib_qp, struct ibv_recv_wr *ib_wr, ++ struct ibv_recv_wr **bad_wr); ++struct ibv_srq *zxdh_ucreate_srq(struct ibv_pd *pd, ++ struct ibv_srq_init_attr *srq_init_attr); ++int zxdh_udestroy_srq(struct ibv_srq *srq); ++int zxdh_umodify_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr, ++ int srq_attr_mask); ++int zxdh_uquery_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr); ++int zxdh_upost_srq_recv(struct ibv_srq *srq, struct ibv_recv_wr *recv_wr, ++ struct ibv_recv_wr **bad_recv_wr); ++int zxdh_uget_srq_num(struct ibv_srq *srq, uint32_t *srq_num); ++struct ibv_ah *zxdh_ucreate_ah(struct ibv_pd *ibpd, struct ibv_ah_attr *attr); ++int zxdh_udestroy_ah(struct ibv_ah *ibah); ++int zxdh_uattach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, ++ uint16_t lid); ++int zxdh_udetach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, ++ uint16_t lid); ++void zxdh_async_event(struct ibv_context *context, ++ struct ibv_async_event *event); ++void zxdh_set_hw_attrs(struct zxdh_hw_attrs *attrs); ++void *zxdh_mmap(int fd, __u32 len, off_t offset); ++void zxdh_munmap(void *map, __u32 len); ++void zxdh_set_debug_mask(void); ++int zxdh_get_write_imm_split_switch(void); ++struct nl_sock *zxdh_rdmanl_socket_alloc(void); ++bool zxdh_get_copy_on_fork(void); ++int zxdh_get_copy_on_fork_cb(struct nl_msg *msg, void *data); ++int zxdh_rdmanl_saw_err_cb(struct sockaddr_nl *nla, struct nlmsgerr *nlerr, ++ void *arg); ++bool zxdh_get_fork_safety(void); ++void *zxdh_alloc_hw_buf(size_t size); ++int zxdh_rdmanl_get_copy_on_fork(struct nl_sock *nl, ++ nl_recvmsg_msg_cb_t cb_func, void *data); ++ ++//IBVERBS_PABI_VERSION 25 ++#if IBVERBS_PABI_VERSION == 25 ++int zxdh_uquery_device(struct ibv_context *context, ++ struct ibv_device_attr *dev_attr); ++#endif ++ ++static inline int ibv_cmd_create_cq_ex_wrap(struct ibv_context *context, ++ struct ibv_cq_init_attr_ex *attr_ex, ++ struct verbs_cq *cq, ++ struct ibv_create_cq_ex *cmd, ++ size_t cmd_size, ++ struct ib_uverbs_ex_create_cq_resp *resp, ++ size_t resp_size) ++{ ++ return ibv_cmd_create_cq_ex(context, attr_ex, cq, cmd, cmd_size, resp, resp_size, 0); ++} ++ ++#endif /* ZXDH_UMAIN_H */ +diff -ruN baseline/providers/zrdma/zxdh_zrdma.h.tmp b/providers/zrdma/zxdh_zrdma.h.tmp +--- a/providers/zrdma/zxdh_zrdma.h.tmp 1970-01-01 08:00:00.000000000 +0800 ++++ b/providers/zrdma/zxdh_zrdma.h.tmp 2026-08-06 15:29:36.932654080 +0800 +@@ -0,0 +1,305 @@ ++/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */ ++/* Copyright (c) 2024 ZTE Corporation. All rights reserved. */ ++#ifndef ZXDH_UMAIN_H ++#define ZXDH_UMAIN_H ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "zxdh_status.h" ++#include "zxdh_verbs.h" ++ ++#define ZXDH_BASE_PUSH_PAGE 1 ++#define ZXDH_U_MINCQ_SIZE 256 ++#define ZXDH_DB_SHADOW_AREA_SIZE 8 ++#define ZXDH_DB_SQ_OFFSET 0x404 ++#define ZXDH_DB_CQ_OFFSET 0x588 ++ ++#define MIN_UDP_SPORT 1024 ++#define MIN_QP_QPN 1 ++ ++enum zxdh_supported_wc_flags { ++ ZXDH_CQ_SUPPORTED_WC_FLAGS = ++ IBV_WC_EX_WITH_BYTE_LEN | IBV_WC_EX_WITH_IMM | ++ IBV_WC_EX_WITH_QP_NUM | IBV_WC_EX_WITH_SRC_QP | ++ IBV_WC_EX_WITH_SLID | IBV_WC_EX_WITH_SL | ++ IBV_WC_EX_WITH_DLID_PATH_BITS | ++ IBV_WC_EX_WITH_COMPLETION_TIMESTAMP_WALLCLOCK | ++ IBV_WC_EX_WITH_COMPLETION_TIMESTAMP, ++}; ++ ++enum { ++ ZXDH_DBG_QP = 1 << 0, ++ ZXDH_DBG_CQ = 1 << 1, ++ ZXDH_DBG_SRQ = 1 << 2, ++ ZXDH_DBG_QUANTA = 1 << 3, ++}; ++extern uint32_t zxdh_debug_mask; ++#define zxdh_dbg(mask, format, arg...) \ ++ do { \ ++ if (mask & zxdh_debug_mask) { \ ++ int tmp = errno; \ ++ (void)fprintf(stdout, "%s:%d: " format, __func__, __LINE__, \ ++ ##arg); \ ++ errno = tmp; \ ++ } \ ++ } while (0) ++ ++#define zxdh_quanta_dbg(format, arg...) \ ++ do { \ ++ if (ZXDH_DBG_QUANTA & zxdh_debug_mask) { \ ++ int tmp = errno; \ ++ (void)fprintf(stderr, "[QUANTA_DEBUG] %s:%d: " format, \ ++ __func__, __LINE__, ##arg); \ ++ errno = tmp; \ ++ } \ ++ } while (0) ++ ++#define zxdh_status_to_errno_log(zxdh_status, format, arg...) \ ++ do { \ ++ errno = zxdh_status_code_to_errno(zxdh_status); \ ++ int tmp = errno; \ ++ (void)fprintf(stdout, "%s:%d: " "[ERROR] status=%s errno=%d: " format"\n", __func__, \ ++ __LINE__, zxdh_status_code_to_string(zxdh_status), errno, ##arg); \ ++ errno = tmp; \ ++ } while (0) ++ ++struct zxdh_udevice { ++ struct verbs_device ibv_dev; ++}; ++ ++struct zxdh_uah { ++ struct ibv_ah ibv_ah; ++ uint32_t ah_id; ++ struct ibv_global_route grh; ++}; ++ ++struct zxdh_upd { ++ struct ibv_pd ibv_pd; ++ void *arm_cq_page; ++ void *arm_cq; ++ uint32_t pd_id; ++}; ++ ++struct zxdh_uvcontext { ++ struct verbs_context ibv_ctx; ++ struct zxdh_upd *iwupd; ++ struct zxdh_dev_attrs dev_attrs; ++ void *db; ++ void *sq_db; ++ void *cq_db; ++ int abi_ver; ++ bool legacy_mode; ++ uint8_t zxdh_write_imm_split_switch; ++ struct zxdh_uvcontext_ops *cxt_ops; ++ void *srq_db; ++ uint32_t max_hw_srq_mmap_size; ++ void *ext_qp_buff_base; ++ void *ext_sq_db; ++ uint8_t is_qp_ext_mem_support; ++}; ++ ++ ++ ++struct zxdh_uqp; ++ ++struct zxdh_cq_buf { ++ struct list_node list; ++ struct zxdh_cq cq; ++ struct verbs_mr vmr; ++}; ++ ++ ++struct zxdh_ucq { ++ struct verbs_cq verbs_cq; ++ struct verbs_mr vmr; ++ struct verbs_mr vmr_shadow_area; ++ pthread_spinlock_t lock; ++ size_t buf_size; ++ bool is_armed; ++ enum zxdh_cmpl_notify last_notify; ++ int comp_vector; ++ uint32_t report_rtt; ++ struct zxdh_uqp *uqp; ++ struct zxdh_cq cq; ++ struct list_head resize_list; ++ /* for extended CQ completion fields */ ++ struct zxdh_cq_poll_info cur_cqe; ++ bool resize_enable; ++ bool is_ext_buf_mode; ++}; ++ ++struct zxdh_usrq { ++ struct ibv_srq ibv_srq; ++ struct verbs_mr vmr; ++ struct verbs_mr list_vmr; ++ struct verbs_mr db_vmr; ++ size_t total_buf_size; ++ size_t buf_size; ++ size_t list_buf_size; ++ size_t db_buf_size; ++ size_t srq_size; ++ size_t srq_list_size; ++ uint32_t srq_id; ++ uint32_t max_wr; ++ uint32_t max_sge; ++ uint32_t srq_limit; ++ pthread_spinlock_t lock; ++ uint32_t wq_size; ++ struct ibv_recv_wr *pend_rx_wr; ++ struct zxdh_srq srq; ++}; ++ ++typedef struct wqe_rate_limit_info { ++ unsigned long long gap_deadline; ++ unsigned int *vhca_wqe_db; ++ int vhca_qp_num; ++ int vhca_rate; ++ unsigned int total_wqe_size; ++ unsigned int iwuqp_qpn; ++ float param1; ++ float param2; ++ int wqe_rate_limit; ++ uint8_t first_wqe_flag; ++ uint8_t current_log2_size; ++ struct wqe_rate_limit_info *next; ++} wqe_rate_limit_info_t; ++ ++struct zxdh_uqp { ++ struct verbs_qp vqp; ++ struct zxdh_ucq *send_cq; ++ struct zxdh_ucq *recv_cq; ++ struct zxdh_usrq *srq; ++ struct verbs_mr vmr; ++ size_t buf_size; ++ uint32_t zxdh_drv_opt; ++ pthread_spinlock_t lock; ++ pthread_spinlock_t quanta_lock; ++ uint16_t sq_sig_all; ++ uint16_t qperr; ++ uint16_t rsvd; ++ uint32_t pending_rcvs; ++ uint32_t wq_size; ++ struct ibv_recv_wr *pend_rx_wr; ++ struct zxdh_qp qp; ++ enum ibv_qp_type qp_type; ++ struct zxdh_sge *recv_sges; ++ unsigned int is_srq:1; ++ unsigned int ext_buf_mode:1; ++ uint8_t inline_data[ZXDH_MAX_INLINE_DATA_SIZE]; ++ wqe_rate_limit_info_t *wqe_rate_limit_info; ++}; ++ ++struct zxdh_umr { ++ struct verbs_mr vmr; ++ uint32_t acc_flags; ++ uint8_t leaf_pbl_size; ++ uint8_t host_page_size; ++ uint64_t mr_pa_pble_index; ++}; ++ ++/* zxdh_verbs.c */ ++int zxdh_uquery_device_ex(struct ibv_context *context, ++ const struct ibv_query_device_ex_input *input, ++ struct ibv_device_attr_ex *attr, size_t attr_size); ++int zxdh_uquery_port(struct ibv_context *context, uint8_t port, ++ struct ibv_port_attr *attr); ++struct ibv_pd *zxdh_ualloc_pd(struct ibv_context *context); ++int zxdh_ufree_pd(struct ibv_pd *pd); ++struct ibv_mr *zxdh_ureg_mr(struct ibv_pd *pd, void *addr, size_t length, ++ uint64_t hca_va, int access); ++ ++struct ibv_mr *zxdh_ureg_mr_dmabuf(struct ibv_pd *pd, uint64_t offset, ++ size_t length, uint64_t iova, int fd, ++ int access); ++ ++int zxdh_udereg_mr(struct verbs_mr *vmr); ++int zxdh_urereg_mr(struct verbs_mr *vmr, int flags, struct ibv_pd *pd, ++ void *addr, size_t length, int access); ++ ++struct ibv_mw *zxdh_ualloc_mw(struct ibv_pd *pd, enum ibv_mw_type type); ++int zxdh_ubind_mw(struct ibv_qp *qp, struct ibv_mw *mw, ++ struct ibv_mw_bind *mw_bind); ++int zxdh_udealloc_mw(struct ibv_mw *mw); ++struct ibv_cq *zxdh_ucreate_cq(struct ibv_context *context, int cqe, ++ struct ibv_comp_channel *channel, ++ int comp_vector); ++struct ibv_cq_ex *zxdh_ucreate_cq_ex(struct ibv_context *context, ++ struct ibv_cq_init_attr_ex *attr_ex); ++void zxdh_ibvcq_ex_fill_priv_funcs(struct zxdh_ucq *iwucq, ++ struct ibv_cq_init_attr_ex *attr_ex); ++int zxdh_uresize_cq(struct ibv_cq *cq, int cqe); ++int zxdh_udestroy_cq(struct ibv_cq *cq); ++int zxdh_umodify_cq(struct ibv_cq *cq, struct ibv_modify_cq_attr *attr); ++int zxdh_upoll_cq(struct ibv_cq *cq, int entries, struct ibv_wc *entry); ++int zxdh_uarm_cq(struct ibv_cq *cq, int solicited); ++void zxdh_cq_event(struct ibv_cq *cq); ++struct ibv_qp *zxdh_ucreate_qp(struct ibv_pd *pd, ++ struct ibv_qp_init_attr *attr); ++struct ibv_qp *zxdh_ucreate_qp_ex(struct ibv_context *context, ++ struct ibv_qp_init_attr_ex *attr); ++int zxdh_uquery_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask, ++ struct ibv_qp_init_attr *init_attr); ++int zxdh_umodify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask); ++int zxdh_udestroy_qp(struct ibv_qp *qp); ++int zxdh_upost_send(struct ibv_qp *ib_qp, struct ibv_send_wr *ib_wr, ++ struct ibv_send_wr **bad_wr); ++int zxdh_upost_recv(struct ibv_qp *ib_qp, struct ibv_recv_wr *ib_wr, ++ struct ibv_recv_wr **bad_wr); ++struct ibv_srq *zxdh_ucreate_srq(struct ibv_pd *pd, ++ struct ibv_srq_init_attr *srq_init_attr); ++int zxdh_udestroy_srq(struct ibv_srq *srq); ++int zxdh_umodify_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr, ++ int srq_attr_mask); ++int zxdh_uquery_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr); ++int zxdh_upost_srq_recv(struct ibv_srq *srq, struct ibv_recv_wr *recv_wr, ++ struct ibv_recv_wr **bad_recv_wr); ++int zxdh_uget_srq_num(struct ibv_srq *srq, uint32_t *srq_num); ++struct ibv_ah *zxdh_ucreate_ah(struct ibv_pd *ibpd, struct ibv_ah_attr *attr); ++int zxdh_udestroy_ah(struct ibv_ah *ibah); ++int zxdh_uattach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, ++ uint16_t lid); ++int zxdh_udetach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, ++ uint16_t lid); ++void zxdh_async_event(struct ibv_context *context, ++ struct ibv_async_event *event); ++void zxdh_set_hw_attrs(struct zxdh_hw_attrs *attrs); ++void *zxdh_mmap(int fd, __u32 len, off_t offset); ++void zxdh_munmap(void *map, __u32 len); ++void zxdh_set_debug_mask(void); ++int zxdh_get_write_imm_split_switch(void); ++struct nl_sock *zxdh_rdmanl_socket_alloc(void); ++bool zxdh_get_copy_on_fork(void); ++int zxdh_get_copy_on_fork_cb(struct nl_msg *msg, void *data); ++int zxdh_rdmanl_saw_err_cb(struct sockaddr_nl *nla, struct nlmsgerr *nlerr, ++ void *arg); ++bool zxdh_get_fork_safety(void); ++void *zxdh_alloc_hw_buf(size_t size); ++int zxdh_rdmanl_get_copy_on_fork(struct nl_sock *nl, ++ nl_recvmsg_msg_cb_t cb_func, void *data); ++ ++//IBVERBS_PABI_VERSION 25 ++#if IBVERBS_PABI_VERSION == 25 ++int zxdh_uquery_device(struct ibv_context *context, ++ struct ibv_device_attr *dev_attr); ++#endif ++ ++static inline int ibv_cmd_create_cq_ex_wrap(struct ibv_context *context, ++ struct ibv_cq_init_attr_ex *attr_ex, ++ struct verbs_cq *cq, ++ struct ibv_create_cq_ex *cmd, ++ size_t cmd_size, ++ struct ib_uverbs_ex_create_cq_resp *resp, ++ size_t resp_size) ++{ ++ return ibv_cmd_create_cq_ex(context, attr_ex, cq, cmd, cmd_size, resp, resp_size, 0); ++} ++ ++#endif /* ZXDH_UMAIN_H */ +--- a/kernel-headers/rdma/zxdh-abi.h 2026-08-06 15:37:55.334861527 +0800 ++++ b/kernel-headers/rdma/zxdh-abi.h 2026-08-06 15:29:26.377929234 +0800 +@@ -6,16 +6,17 @@ + + #include + +-/* zxdh must support legacy GEN_1 i40iw kernel +- * and user-space whose last ABI ver is 5 +- */ ++/* user-space whose last ABI ver is 5 */ + #define ZXDH_ABI_VER 5 +- ++#define ZXDH_CONTEXT_VER_V1 5 ++#define ZXDH_CONTEXT_VER_V2 6 ++#define ZXDH_CONTEXT_VER_V3 7 + enum zxdh_memreg_type { + ZXDH_MEMREG_TYPE_MEM = 0, + ZXDH_MEMREG_TYPE_QP = 1, + ZXDH_MEMREG_TYPE_CQ = 2, + ZXDH_MEMREG_TYPE_SRQ = 3, ++ ZXDH_MEMREG_TYPE_EXTERNEL_MEM = 4, + }; + + enum zxdh_db_addr_type { +@@ -35,7 +36,7 @@ + __u32 wq_size; /* size of the WQs (SQ+RQ) in the mmaped area */ + __u8 kernel_ver; + __u8 db_addr_type; +- __u8 rsvd[2]; ++ __u16 rdma_tool_flags; + __aligned_u64 feature_flags; + __aligned_u64 sq_db_mmap_key; + __aligned_u64 cq_db_mmap_key; +@@ -51,8 +52,19 @@ + __u32 min_hw_cq_size; + __u32 max_hw_cq_size; + __u16 max_hw_sq_chunk; +- __u8 hw_rev; +- __u8 rsvd2; ++ __u8 rsvd; ++ __u8 chip_rev; ++ __aligned_u64 srq_db_mmap_key; ++ __aligned_u64 sq_db_bar_off; ++ __aligned_u64 cq_db_bar_off; ++ __aligned_u64 srq_db_bar_off; ++ __u32 srq_db_mmap_size; ++ __u16 rsv[6]; ++ __aligned_u64 ext_sq_db_mmap_key; ++ __aligned_u64 l2d_addr_mmap_key; ++ __u32 max_hw_rq_sges; ++ __u8 rsvd1[3]; ++ __u8 is_qp_ext_mem_support; + }; + + struct zxdh_alloc_pd_resp { +@@ -67,11 +79,18 @@ + struct zxdh_create_cq_req { + __aligned_u64 user_cq_buf; + __aligned_u64 user_shadow_area; ++ __aligned_u64 dev_cq_buf; ++ __aligned_u64 dev_shadow_area; + }; + + struct zxdh_create_qp_req { + __aligned_u64 user_wqe_bufs; + __aligned_u64 user_compl_ctx; ++ __aligned_u64 dev_wqe_bufs; ++ __u8 rsvd[7]; ++ __u8 support_ext_qp; ++ __u32 rq_offset; ++ __u32 shadow_offset; + }; + + struct zxdh_create_srq_req { +@@ -82,13 +101,13 @@ + }; + + struct zxdh_mem_reg_req { +- __u16 reg_type; /* enum zxdh_memreg_type */ +- __u16 cq_pages; +- __u16 rq_pages; +- __u16 sq_pages; +- __u16 srq_pages; ++ __u32 reg_type; /* enum zxdh_memreg_type */ ++ __u32 cq_pages; ++ __u32 rq_pages; ++ __u32 sq_pages; ++ __u32 srq_pages; + __u16 srq_list_pages; +- __u8 rsvd[4]; ++ __u8 rsvd[2]; + }; + + struct zxdh_reg_mr_resp { +@@ -117,14 +136,21 @@ + __u32 zxdh_drv_opt; + __u16 push_idx; + __u8 lsmm; +- __u8 rsvd; ++ __u8 wqe_rate_limit_flag : 1; ++ __u8 wqe_rate_limit_valid : 1; ++ __u8 rsvd : 6; + __u32 qp_caps; ++ __u8 rsvd2[7]; ++ __u8 is_ext_qp; ++ __aligned_u64 sq_offset; ++ __aligned_u64 rq_offset; + }; + + struct zxdh_create_srq_resp { + __u32 srq_id; + __u32 actual_srq_size; + __u32 actual_srq_list_size; ++ __u32 base_srqn; + __u8 rsvd[4]; + }; + diff --git a/rdma-core.spec b/rdma-core.spec index 771890c399e2213779cf75e132b8d6940a1583e7..e664af0ea7107c5c030758ca8bfcf5a8416c8cfc 100644 --- a/rdma-core.spec +++ b/rdma-core.spec @@ -1,13 +1,14 @@ Summary: Userspace components for Linux Kernel's drivers/infiniband subsystem Name: rdma-core Version: 47.0 -Release: 7%{?dist} +Release: 8%{?dist} License: GPLv2 or BSD Url: https://github.com/linux-rdma/rdma-core Source0: %{url}/releases/download/v%{version}/%{name}-%{version}.tar.gz Patch3000: 0001-udev-keep-NAME_KERNEL-as-default-interface-naming-co.patch Patch3001: 0002-add-ZTE-Dinghai-RDMA-driver.patch Patch3002: 0003-add-SXE2-RDMA-driver.patch +Patch3003: 0004-update-ZTE-Dinghai-RDMA-driver.patch BuildRequires: binutils gcc pkgconfig pkgconfig(libnl-3.0) pkgconfig(libnl-route-3.0) BuildRequires: cmake >= 2.8.11 BuildRequires: ninja-build @@ -527,6 +528,10 @@ fi %changelog +* Thu Aug 06 2026 update ZTE Dinghai RDMA driver - 47.0-8 +- [Type] enhancement +- [DESC] Update ZTE Dinghai RDMA driver with complete implementation + * Mon May 25 2026 add SXE2 RDMA driver - 47.0-7 - [type] enhancement - [DESC] add SXE2 RDMA driver