diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.c b/drivers/infiniband/hw/hns/hns_roce_bond.c index 0fc026eb40e8882dc3828534ad33499bc87e8ce7..7adae8990acd2748e7f5c18ededdec251ca7fa29 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.c +++ b/drivers/infiniband/hw/hns/hns_roce_bond.c @@ -93,10 +93,16 @@ bool hns_roce_bond_is_active(struct hns_roce_dev *hr_dev) static inline bool is_active_slave(struct net_device *net_dev, struct hns_roce_bond_group *bond_grp) { + struct net_device *slave_dev; + if (!bond_grp || !bond_grp->bond || !bond_grp->bond->curr_active_slave) return false; - return net_dev == bond_grp->bond->curr_active_slave->dev; + rcu_read_lock(); + slave_dev = bond_option_active_slave_get_rcu(bond_grp->bond); + rcu_read_unlock(); + + return net_dev == slave_dev; } struct net_device *hns_roce_get_bond_netdev(struct hns_roce_dev *hr_dev) diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index 6bb5f4b6c7c23653546980cfce83f5734b6a4ecc..7cda55debe62dc06741ffa4410d3417f64a76c3e 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -37,6 +37,47 @@ #include "hns_roce_hem.h" #include "hns_roce_common.h" +void hns_roce_put_cq_bankid_for_uctx(struct hns_roce_ucontext *uctx) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(uctx->ibucontext.device); + struct hns_roce_cq_table *cq_table = &hr_dev->cq_table; + + if (hr_dev->pci_dev->revision < PCI_REVISION_ID_HIP09) + return; + + mutex_lock(&cq_table->bank_mutex); + cq_table->ctx_num[uctx->cq_bank_id]--; + mutex_unlock(&cq_table->bank_mutex); +} + +void hns_roce_get_cq_bankid_for_uctx(struct hns_roce_ucontext *uctx) +{ +#define INVALID_LOAD_CQNUM 0xFFFFFFFF + struct hns_roce_dev *hr_dev = to_hr_dev(uctx->ibucontext.device); + struct hns_roce_cq_table *cq_table = &hr_dev->cq_table; + u32 least_load = INVALID_LOAD_CQNUM; + u8 bankid = 0; + u8 i; + + if (hr_dev->pci_dev->revision < PCI_REVISION_ID_HIP09) + return; + + mutex_lock(&cq_table->bank_mutex); + for (i = 0; i < HNS_ROCE_CQ_BANK_NUM; i++) { + if (!(cq_table->valid_cq_bank_mask & BIT(i))) + continue; + + if (cq_table->ctx_num[i] < least_load) { + least_load = cq_table->ctx_num[i]; + bankid = i; + } + } + cq_table->ctx_num[bankid]++; + mutex_unlock(&cq_table->bank_mutex); + + uctx->cq_bank_id = bankid; +} + static u8 get_least_load_bankid_for_cq(struct hns_roce_bank *bank) { u32 least_load = bank[0].inuse; @@ -55,7 +96,21 @@ static u8 get_least_load_bankid_for_cq(struct hns_roce_bank *bank) return bankid; } -static int alloc_cqn(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) +static u8 select_cq_bankid(struct hns_roce_dev *hr_dev, struct hns_roce_bank *bank, + struct ib_udata *udata) +{ + struct hns_roce_ucontext *uctx = udata ? + rdma_udata_to_drv_context(udata, struct hns_roce_ucontext, + ibucontext) : NULL; + /* only HIP08 is not applied now, and use bank 0 for kernel */ + if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) + return uctx ? uctx->cq_bank_id : 0; + + return get_least_load_bankid_for_cq(bank); +} + +static int alloc_cqn(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, + struct ib_udata *udata) { struct hns_roce_cq_table *cq_table = &hr_dev->cq_table; struct hns_roce_bank *bank; @@ -63,7 +118,7 @@ static int alloc_cqn(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) int id; mutex_lock(&cq_table->bank_mutex); - bankid = get_least_load_bankid_for_cq(cq_table->bank); + bankid = select_cq_bankid(hr_dev, cq_table->bank, udata); bank = &cq_table->bank[bankid]; id = ida_alloc_range(&bank->ida, bank->min, bank->max, GFP_KERNEL); @@ -178,12 +233,11 @@ static void free_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_CQC, hr_cq->cqn); - if (ret) + if (ret) { + hr_cq->delayed_destroy_flag = true; dev_err_ratelimited(dev, "DESTROY_CQ failed (%d) for CQN %06lx\n", ret, hr_cq->cqn); - - if (ret == -EBUSY) - hr_cq->delayed_destroy_flag = true; + } xa_erase(&cq_table->array, hr_cq->cqn); xa_erase_irq(&cq_table->array, hr_cq->cqn); @@ -417,7 +471,7 @@ int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr, goto err_cq_buf; } - ret = alloc_cqn(hr_dev, hr_cq); + ret = alloc_cqn(hr_dev, hr_cq, udata); if (ret) { ibdev_err(ibdev, "failed to alloc CQN, ret = %d.\n", ret); goto err_cq_db; @@ -550,6 +604,11 @@ void hns_roce_init_cq_table(struct hns_roce_dev *hr_dev) cq_table->bank[i].max = hr_dev->caps.num_cqs / HNS_ROCE_CQ_BANK_NUM - 1; } + + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_LIMIT_BANK) + cq_table->valid_cq_bank_mask = VALID_CQ_BANK_MASK_LIMIT; + else + cq_table->valid_cq_bank_mask = VALID_CQ_BANK_MASK_DEFAULT; } void hns_roce_cleanup_cq_table(struct hns_roce_dev *hr_dev) diff --git a/drivers/infiniband/hw/hns/hns_roce_dca.c b/drivers/infiniband/hw/hns/hns_roce_dca.c index 4cef4159179533a10939e0629dc0d6a5bcd2aef1..eb408130329bded5fd851132f5a0e7d2484680d4 100644 --- a/drivers/infiniband/hw/hns/hns_roce_dca.c +++ b/drivers/infiniband/hw/hns/hns_roce_dca.c @@ -309,6 +309,33 @@ hr_qp_to_dca_ctx(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) return to_hr_dca_ctx(hr_dev, uctx); } +int hns_roce_map_dca_safe_page(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp) +{ + unsigned int page_count = hr_qp->dca_cfg.npages; + struct ib_device *ibdev = &hr_dev->ib_dev; + dma_addr_t *pages; + unsigned int i; + int ret; + + pages = kvcalloc(page_count, sizeof(dma_addr_t), GFP_KERNEL); + if (IS_ERR_OR_NULL(pages)) { + ibdev_err(ibdev, "failed to alloc DCA safe page array.\n"); + return -ENOMEM; + } + + for (i = 0; i < page_count; i++) + pages[i] = hr_dev->dca_safe_page; + + ret = hns_roce_mtr_map(hr_dev, &hr_qp->mtr, pages, page_count); + if (ret) + ibdev_err(ibdev, "failed to map safe page for DCA, ret = %d.\n", + ret); + + kvfree(pages); + return ret; +} + static int config_dca_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, dma_addr_t *pages, int page_count) @@ -335,6 +362,29 @@ static int config_dca_qpc(struct hns_roce_dev *hr_dev, return 0; } +static int config_dca_qpc_to_safe_page(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp) +{ + unsigned int page_count = hr_qp->dca_cfg.npages; + dma_addr_t *pages; + unsigned int i; + int ret; + + might_sleep(); + + pages = kvcalloc(page_count, sizeof(dma_addr_t), GFP_KERNEL); + if (IS_ERR_OR_NULL(pages)) + return -ENOMEM; + + for (i = 0; i < page_count; i++) + pages[i] = hr_dev->dca_safe_page; + + ret = config_dca_qpc(hr_dev, hr_qp, pages, page_count); + + kvfree(pages); + return ret; +} + static int setup_dca_buf_to_hw(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, struct hns_roce_dca_ctx *ctx, u32 buf_id, @@ -567,7 +617,7 @@ static int active_dca_pages_proc(struct dca_mem *mem, int index, void *param) } for (; changed && i < mem->page_count; i++) - if (dca_page_is_free(state)) + if (dca_page_is_free(&mem->states[i])) free_pages++; /* Clean mem changed to dirty */ @@ -972,7 +1022,7 @@ static void process_aging_dca_mem(struct hns_roce_dev *hr_dev, list_for_each_entry_safe(cfg, tmp_cfg, &ctx->aging_new_list, aging_node) list_move(&cfg->aging_node, &ctx->aging_proc_list); - while (!ctx->exit_aging && !list_empty(&ctx->aging_proc_list)) { + while (!ctx->exit_aging && !list_empty_careful(&ctx->aging_proc_list)) { cfg = list_first_entry(&ctx->aging_proc_list, struct hns_roce_dca_cfg, aging_node); list_del_init_careful(&cfg->aging_node); @@ -980,8 +1030,10 @@ static void process_aging_dca_mem(struct hns_roce_dev *hr_dev, spin_unlock(&ctx->aging_lock); if (start_free_dca_buf(ctx, cfg->dcan)) { - if (hr_dev->hw->chk_dca_buf_inactive(hr_dev, hr_qp)) - free_buf_from_dca_mem(ctx, cfg); + if (hr_dev->hw->chk_dca_buf_inactive(hr_dev, hr_qp)) { + if (!config_dca_qpc_to_safe_page(hr_dev, hr_qp)) + free_buf_from_dca_mem(ctx, cfg); + } stop_free_dca_buf(ctx, cfg->dcan); } diff --git a/drivers/infiniband/hw/hns/hns_roce_dca.h b/drivers/infiniband/hw/hns/hns_roce_dca.h index 7733887ce5e176011d4ff6d765169ce22f4c1632..36f03f5357d7c245dc5cfcb0ded6de30b618922b 100644 --- a/drivers/infiniband/hw/hns/hns_roce_dca.h +++ b/drivers/infiniband/hw/hns/hns_roce_dca.h @@ -75,4 +75,6 @@ void hns_roce_modify_dca(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, void hns_roce_enum_dca_pool(struct hns_roce_dca_ctx *dca_ctx, void *param, hns_dca_enum_callback cb); +int hns_roce_map_dca_safe_page(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp); #endif diff --git a/drivers/infiniband/hw/hns/hns_roce_debugfs.c b/drivers/infiniband/hw/hns/hns_roce_debugfs.c index 3c2d7096fe1353bc39e04a841a03ff6919e8f4be..7023c3cefaa7dc12fe9ddebcb5a2685d66276b5b 100644 --- a/drivers/infiniband/hw/hns/hns_roce_debugfs.c +++ b/drivers/infiniband/hw/hns/hns_roce_debugfs.c @@ -187,8 +187,8 @@ static void dca_setup_pool_name(pid_t pid, bool is_kdca, char *name, int size) static u64 calc_loading_percent(size_t total, size_t free, u32 *out_rem) { - u32 all_pages, used_pages, free_pages, scale; - u64 percent = 0; + u64 used_pages, scale, all_pages, free_pages; + u64 percent = U64_MAX; u32 rem = 0; all_pages = total >> HNS_HW_PAGE_SHIFT; @@ -214,6 +214,9 @@ static void dca_print_pool_stats(struct hns_roce_dca_ctx *ctx, pid_t pid, u32 rem = 0; percent = calc_loading_percent(ctx->total_size, ctx->free_size, &rem); + if (percent == U64_MAX) + return; + dca_setup_pool_name(pid, is_kdca, name, sizeof(name)); seq_printf(file, "%-10s %-16ld %-16ld %-16u %llu.%0*u\n", name, ctx->total_size / KB, ctx->free_size / KB, ctx->free_mems, @@ -366,6 +369,9 @@ static void dca_stats_ctx_mem_in_seqfile(struct hns_roce_dca_ctx *ctx, dca_ctx_stats_mem(ctx, &stats); percent = calc_loading_percent(stats.total_size, stats.free_size, &rem); + if (percent == U64_MAX) + return; + seq_printf(file, DCA_STAT_NAME_FMT "%llu.%0*u\n", "Loading:", percent, LOADING_PERCENT_SHIFT, rem); dca_ctx_print_mem_kb(file, "Total:", stats.total_size); diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index fdc1fe5e6a811ac86138ce40c005152a384cd4fa..85e160132135c0bbfe8bea74bbc61ad8bb6e9383 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -105,6 +105,10 @@ #define CQ_BANKID_SHIFT 2 #define CQ_BANKID_MASK GENMASK(1, 0) +#define VALID_CQ_BANK_MASK_DEFAULT 0xF +#define VALID_CQ_BANK_MASK_LIMIT 0x9 + +#define VALID_EXT_SGE_QP_BANK_MASK_LIMIT 0x41 #define HNS_ROCE_MAX_CQ_COUNT 0xFFFF #define HNS_ROCE_MAX_CQ_PERIOD 0xFFFF @@ -168,6 +172,7 @@ enum { HNS_ROCE_CAP_FLAG_CQE_INLINE = BIT(19), HNS_ROCE_CAP_FLAG_BOND = BIT(21), HNS_ROCE_CAP_FLAG_SRQ_RECORD_DB = BIT(22), + HNS_ROCE_CAP_FLAG_LIMIT_BANK = BIT(23), }; #define HNS_ROCE_DB_TYPE_COUNT 2 @@ -265,6 +270,7 @@ struct hns_roce_ucontext { struct list_head list; /* link all uctx to uctx_list on hr_dev */ pid_t pid; /* process id to which the uctx belongs */ struct hns_dca_ctx_debugfs dca_dbgfs; + u8 cq_bank_id; }; struct hns_roce_pd { @@ -592,6 +598,8 @@ struct hns_roce_cq_table { struct hns_roce_hem_table table; struct hns_roce_bank bank[HNS_ROCE_CQ_BANK_NUM]; struct mutex bank_mutex; + u32 ctx_num[HNS_ROCE_CQ_BANK_NUM]; + u8 valid_cq_bank_mask; }; struct hns_roce_srq_table { @@ -1076,11 +1084,13 @@ struct hns_roce_hw { #define HNS_ROCE_SCC_PARAM_SIZE 4 struct hns_roce_scc_param { __le32 param[HNS_ROCE_SCC_PARAM_SIZE]; - u32 lifespan; + __le32 lifespan; unsigned long timestamp; enum hns_roce_scc_algo algo_type; struct delayed_work scc_cfg_dwork; struct hns_roce_dev *hr_dev; + __le32 latest_param[HNS_ROCE_SCC_PARAM_SIZE]; + struct mutex scc_mutex; /* protect @param and @lastest_param */ }; struct hns_roce_dev { @@ -1161,6 +1171,9 @@ struct hns_roce_dev { struct mutex mtr_unfree_list_mutex; /* protect mtr_unfree_list */ struct list_head umem_unfree_list; /* list of unfree umem on this dev */ struct mutex umem_unfree_list_mutex; /* protect umem_unfree_list */ + + void *dca_safe_buf; + dma_addr_t dca_safe_page; }; static inline struct hns_roce_dev *to_hr_dev(struct ib_device *ib_dev) @@ -1471,4 +1484,6 @@ void hns_roce_add_unfree_mtr(struct hns_roce_mtr_node *pos, void hns_roce_free_unfree_mtr(struct hns_roce_dev *hr_dev); int hns_roce_alloc_scc_param(struct hns_roce_dev *hr_dev); void hns_roce_dealloc_scc_param(struct hns_roce_dev *hr_dev); +void hns_roce_put_cq_bankid_for_uctx(struct hns_roce_ucontext *uctx); +void hns_roce_get_cq_bankid_for_uctx(struct hns_roce_ucontext *uctx); #endif /* _HNS_ROCE_DEVICE_H */ diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 3f254ac48b425e9c82ad5adc9906d820f8fc240f..4f595e6a4443a50bf6e6aeb7ea749710f711c998 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -5974,10 +5974,12 @@ static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, /* Modify qp to reset before destroying qp */ ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0, hr_qp->state, IB_QPS_RESET, udata); - if (ret) + if (ret) { + hr_qp->delayed_destroy_flag = true; ibdev_err_ratelimited(ibdev, "failed to modify QP to RST, ret = %d.\n", ret); + } } send_cq = hr_qp->ibqp.send_cq ? to_hr_cq(hr_qp->ibqp.send_cq) : NULL; @@ -6024,9 +6026,6 @@ int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) "failed to destroy QP, QPN = 0x%06lx, ret = %d.\n", hr_qp->qpn, ret); - if (ret == -EBUSY) - hr_qp->delayed_destroy_flag = true; - hns_roce_qp_destroy(hr_dev, hr_qp, udata); return 0; @@ -7259,14 +7258,23 @@ static int hns_roce_v2_config_scc_param(struct hns_roce_dev *hr_dev, hns_roce_cmq_setup_basic_desc(&desc, scc_opcode[algo], false); scc_param = &hr_dev->scc_param[algo]; + mutex_lock(&scc_param->scc_mutex); memcpy(&desc.data, scc_param, sizeof(scc_param->param)); ret = hns_roce_cmq_send(hr_dev, &desc, 1); - if (ret) + if (ret) { ibdev_err_ratelimited(&hr_dev->ib_dev, "failed to configure scc param, opcode: 0x%x, ret = %d.\n", le16_to_cpu(desc.opcode), ret); - return ret; + mutex_unlock(&scc_param->scc_mutex); + return ret; + } + + memcpy(scc_param->latest_param, &desc.data, + sizeof(scc_param->latest_param)); + mutex_unlock(&scc_param->scc_mutex); + + return 0; } static int hns_roce_v2_query_scc_param(struct hns_roce_dev *hr_dev, @@ -7294,7 +7302,11 @@ static int hns_roce_v2_query_scc_param(struct hns_roce_dev *hr_dev, } scc_param = &hr_dev->scc_param[algo]; - memcpy(scc_param, &desc.data, sizeof(scc_param->param)); + mutex_lock(&scc_param->scc_mutex); + memcpy(scc_param->param, &desc.data, sizeof(scc_param->param)); + memcpy(scc_param->latest_param, &desc.data, + sizeof(scc_param->latest_param)); + mutex_unlock(&scc_param->scc_mutex); return 0; } diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index d488c3d5986f497864453b2ce910694a13bd3e5d..9fa62d31d50051fdd95fe4420df25ea14f9a8924 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -262,6 +262,11 @@ static int hns_roce_query_device(struct ib_device *ib_dev, props->max_srq_sge = hr_dev->caps.max_srq_sges; } + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_LIMIT_BANK) { + props->max_cq >>= 1; + props->max_qp >>= 1; + } + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_FRMR && hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) { props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS; @@ -598,6 +603,7 @@ static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx, mutex_unlock(&hr_dev->uctx_list_mutex); hns_roce_register_uctx_debugfs(hr_dev, context); + hns_roce_get_cq_bankid_for_uctx(context); return 0; @@ -634,6 +640,7 @@ static void hns_roce_dealloc_ucontext(struct ib_ucontext *ibcontext) hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) mutex_destroy(&context->page_mutex); + hns_roce_put_cq_bankid_for_uctx(context); hns_roce_unregister_uctx_debugfs(context); hns_roce_unregister_udca(hr_dev, context); @@ -1370,6 +1377,17 @@ static void hns_roce_dealloc_dfx_cnt(struct hns_roce_dev *hr_dev) kvfree(hr_dev->dfx_cnt); } +static void hns_roce_free_dca_safe_buf(struct hns_roce_dev *hr_dev) +{ + if (!hr_dev->dca_safe_buf) + return; + + dma_free_coherent(hr_dev->dev, PAGE_SIZE, hr_dev->dca_safe_buf, + hr_dev->dca_safe_page); + hr_dev->dca_safe_page = 0; + hr_dev->dca_safe_buf = NULL; +} + int hns_roce_init(struct hns_roce_dev *hr_dev) { struct device *dev = hr_dev->dev; @@ -1483,6 +1501,8 @@ void hns_roce_exit(struct hns_roce_dev *hr_dev, bool bond_cleanup) hns_roce_dealloc_scc_param(hr_dev); hns_roce_unregister_debugfs(hr_dev); + hns_roce_free_dca_safe_buf(hr_dev); + if (hr_dev->hw->hw_exit) hr_dev->hw->hw_exit(hr_dev); hns_roce_free_unfree_umem(hr_dev); diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c index 1f29377b52b87feac4f28d1e4db3dee01784dc47..5c4b6c4f4ca708ba20306407d88c389d802cda5d 100644 --- a/drivers/infiniband/hw/hns/hns_roce_mr.c +++ b/drivers/infiniband/hw/hns/hns_roce_mr.c @@ -153,11 +153,11 @@ static void hns_roce_mr_free(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_MPT, key_to_hw_index(mr->key) & (hr_dev->caps.num_mtpts - 1)); - if (ret) + if (ret) { + mr->delayed_destroy_flag = true; ibdev_warn_ratelimited(ibdev, "failed to destroy mpt, ret = %d.\n", ret); - if (ret == -EBUSY) - mr->delayed_destroy_flag = true; + } } free_mr_pbl(hr_dev, mr); diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index 5ed2647567aa6a8a31d303fb785444e88b299eb3..c75706bc4212757558a1623d7379367b298b9b7c 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -198,22 +198,16 @@ static u8 get_affinity_cq_bank(u8 qp_bank) return (qp_bank >> 1) & CQ_BANKID_MASK; } -static u8 get_least_load_bankid_for_qp(struct ib_qp_init_attr *init_attr, - struct hns_roce_bank *bank) +static u8 get_least_load_bankid_for_qp(struct hns_roce_bank *bank, u8 valid_qp_bank_mask) { #define INVALID_LOAD_QPNUM 0xFFFFFFFF - struct ib_cq *scq = init_attr->send_cq; u32 least_load = INVALID_LOAD_QPNUM; - unsigned long cqn = 0; u8 bankid = 0; u32 bankcnt; u8 i; - if (scq) - cqn = to_hr_cq(scq)->cqn; - for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++) { - if (scq && (get_affinity_cq_bank(i) != (cqn & CQ_BANKID_MASK))) + if (!(valid_qp_bank_mask & BIT(i))) continue; bankcnt = bank[i].inuse; @@ -247,6 +241,42 @@ static int alloc_qpn_with_bankid(struct hns_roce_bank *bank, u8 bankid, return 0; } + +static bool use_ext_sge(struct ib_qp_init_attr *init_attr) +{ + return init_attr->cap.max_send_sge > HNS_ROCE_SGE_IN_WQE || + init_attr->qp_type == IB_QPT_UD || + init_attr->qp_type == IB_QPT_GSI; +} + +static u8 select_qp_bankid(struct hns_roce_dev *hr_dev, + struct ib_qp_init_attr *init_attr) +{ + struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; + struct hns_roce_bank *bank = qp_table->bank; + struct ib_cq *scq = init_attr->send_cq; + u8 valid_qp_bank_mask = 0; + unsigned long cqn = 0; + u8 i; + + if (scq) + cqn = to_hr_cq(scq)->cqn; + + for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++) { + if (scq && (get_affinity_cq_bank(i) != (cqn & CQ_BANKID_MASK))) + continue; + + if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_LIMIT_BANK) && + use_ext_sge(init_attr) && + !(VALID_EXT_SGE_QP_BANK_MASK_LIMIT & BIT(i))) + continue; + + valid_qp_bank_mask |= BIT(i); + } + + return get_least_load_bankid_for_qp(bank, valid_qp_bank_mask); +} + static int alloc_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, struct ib_qp_init_attr *init_attr) { @@ -259,8 +289,7 @@ static int alloc_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, num = 1; } else { mutex_lock(&qp_table->bank_mutex); - bankid = get_least_load_bankid_for_qp(init_attr, qp_table->bank); - + bankid = select_qp_bankid(hr_dev, init_attr); ret = alloc_qpn_with_bankid(&qp_table->bank[bankid], bankid, &num); if (ret) { @@ -668,7 +697,6 @@ static int set_user_sq_size(struct hns_roce_dev *hr_dev, hr_qp->sq.wqe_shift = ucmd->log_sq_stride; hr_qp->sq.wqe_cnt = cnt; - cap->max_send_sge = hr_qp->sq.max_gs; return 0; } @@ -780,7 +808,6 @@ static int set_kernel_sq_size(struct hns_roce_dev *hr_dev, /* sync the parameters of kernel QP to user's configuration */ cap->max_send_wr = cnt; - cap->max_send_sge = hr_qp->sq.max_gs; return 0; } @@ -844,6 +871,8 @@ static int alloc_wqe_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, hns_roce_disable_dca(hr_dev, hr_qp, udata); kvfree(hr_qp->mtr_node); hr_qp->mtr_node = NULL; + } else if (dca_en) { + ret = hns_roce_map_dca_safe_page(hr_dev, hr_qp); } return ret; @@ -864,6 +893,21 @@ static void free_wqe_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, hns_roce_disable_dca(hr_dev, hr_qp, udata); } +static int alloc_dca_safe_page(struct hns_roce_dev *hr_dev) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + + hr_dev->dca_safe_buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE, + &hr_dev->dca_safe_page, + GFP_KERNEL); + if (!hr_dev->dca_safe_buf) { + ibdev_err(ibdev, "failed to alloc dca safe page.\n"); + return -ENOMEM; + } + + return 0; +} + static int alloc_qp_wqe(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, struct ib_qp_init_attr *init_attr, struct ib_udata *udata, @@ -882,6 +926,12 @@ static int alloc_qp_wqe(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, dca_en = check_dca_is_enable(hr_dev, hr_qp, init_attr, !!udata, ucmd->buf_addr); + if (dca_en && !hr_dev->dca_safe_buf) { + ret = alloc_dca_safe_page(hr_dev); + if (ret) + return ret; + } + ret = set_wqe_buf_attr(hr_dev, hr_qp, dca_en, page_shift, &buf_attr); if (ret) { ibdev_err(ibdev, "failed to split WQE buf, ret = %d.\n", ret); diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c index 0a8e714312463b5bb7c986cd711446c05b116f22..0ab99aa9f9d50e9d6c359b6cc5e85c508cb3792d 100644 --- a/drivers/infiniband/hw/hns/hns_roce_srq.c +++ b/drivers/infiniband/hw/hns/hns_roce_srq.c @@ -149,12 +149,12 @@ static void free_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_SRQ, srq->srqn); - if (ret) + if (ret) { + srq->delayed_destroy_flag = true; dev_err_ratelimited(hr_dev->dev, "DESTROY_SRQ failed (%d) for SRQN %06lx\n", ret, srq->srqn); - if (ret == -EBUSY) - srq->delayed_destroy_flag = true; + } xa_erase_irq(&srq_table->xa, srq->srqn); diff --git a/drivers/infiniband/hw/hns/hns_roce_sysfs.c b/drivers/infiniband/hw/hns/hns_roce_sysfs.c index d36f05ac5f1e73c0e69ab452225fae33e0505b58..0ccc75ccb434ebf1f96e5136806ec0fb7c1648b2 100644 --- a/drivers/infiniband/hw/hns/hns_roce_sysfs.c +++ b/drivers/infiniband/hw/hns/hns_roce_sysfs.c @@ -46,6 +46,7 @@ int hns_roce_alloc_scc_param(struct hns_roce_dev *hr_dev) for (i = 0; i < HNS_ROCE_SCC_ALGO_TOTAL; i++) { scc_param[i].algo_type = i; scc_param[i].hr_dev = hr_dev; + mutex_init(&scc_param[i].scc_mutex); INIT_DELAYED_WORK(&scc_param[i].scc_cfg_dwork, scc_param_config_work); } @@ -63,8 +64,10 @@ void hns_roce_dealloc_scc_param(struct hns_roce_dev *hr_dev) if (!hr_dev->scc_param) return; - for (i = 0; i < HNS_ROCE_SCC_ALGO_TOTAL; i++) + for (i = 0; i < HNS_ROCE_SCC_ALGO_TOTAL; i++) { cancel_delayed_work_sync(&hr_dev->scc_param[i].scc_cfg_dwork); + mutex_destroy(&hr_dev->scc_param[i].scc_mutex); + } kvfree(hr_dev->scc_param); hr_dev->scc_param = NULL; @@ -110,7 +113,13 @@ static ssize_t scc_attr_show(struct ib_device *ibdev, u32 port_num, scc_param = &hr_dev->scc_param[scc_attr->algo_type]; - memcpy(&val, (void *)scc_param + scc_attr->offset, scc_attr->size); + mutex_lock(&scc_param->scc_mutex); + if (scc_attr->offset == offsetof(typeof(*scc_param), lifespan)) + val = scc_param->lifespan; + else + memcpy(&val, (void *)scc_param->latest_param + scc_attr->offset, + scc_attr->size); + mutex_unlock(&scc_param->scc_mutex); return sysfs_emit(buf, "%u\n", le32_to_cpu(val)); } @@ -141,14 +150,16 @@ static ssize_t scc_attr_store(struct ib_device *ibdev, u32 port_num, attr_val = cpu_to_le32(val); scc_param = &hr_dev->scc_param[scc_attr->algo_type]; + mutex_lock(&scc_param->scc_mutex); memcpy((void *)scc_param + scc_attr->offset, &attr_val, scc_attr->size); + mutex_unlock(&scc_param->scc_mutex); /* lifespan is only used for driver */ if (scc_attr->offset >= offsetof(typeof(*scc_param), lifespan)) return count; - lifespan_jiffies = msecs_to_jiffies(scc_param->lifespan); + lifespan_jiffies = msecs_to_jiffies(le32_to_cpu(scc_param->lifespan)); exp_time = scc_param->timestamp + lifespan_jiffies; if (time_is_before_eq_jiffies(exp_time)) { @@ -193,11 +204,11 @@ static umode_t scc_attr_is_visible(struct kobject *kobj, .max = _max, \ } -#define HNS_PORT_DCQCN_CC_ATTR_RW(_name, NAME) \ - struct hns_port_cc_attr hns_roce_port_attr_dcqcn_##_name = \ - __HNS_SCC_ATTR(_name, HNS_ROCE_SCC_ALGO_DCQCN, \ - HNS_ROCE_DCQCN_##NAME##_OFS, \ - HNS_ROCE_DCQCN_##NAME##_SZ, \ +#define HNS_PORT_DCQCN_CC_ATTR_RW(_name, NAME) \ + static struct hns_port_cc_attr hns_roce_port_attr_dcqcn_##_name = \ + __HNS_SCC_ATTR(_name, HNS_ROCE_SCC_ALGO_DCQCN, \ + HNS_ROCE_DCQCN_##NAME##_OFS, \ + HNS_ROCE_DCQCN_##NAME##_SZ, \ 0, HNS_ROCE_DCQCN_##NAME##_MAX) HNS_PORT_DCQCN_CC_ATTR_RW(ai, AI); @@ -233,11 +244,11 @@ static const struct attribute_group dcqcn_cc_param_group = { .is_visible = scc_attr_is_visible, }; -#define HNS_PORT_LDCP_CC_ATTR_RW(_name, NAME) \ - struct hns_port_cc_attr hns_roce_port_attr_ldcp_##_name = \ - __HNS_SCC_ATTR(_name, HNS_ROCE_SCC_ALGO_LDCP, \ - HNS_ROCE_LDCP_##NAME##_OFS, \ - HNS_ROCE_LDCP_##NAME##_SZ, \ +#define HNS_PORT_LDCP_CC_ATTR_RW(_name, NAME) \ + static struct hns_port_cc_attr hns_roce_port_attr_ldcp_##_name = \ + __HNS_SCC_ATTR(_name, HNS_ROCE_SCC_ALGO_LDCP, \ + HNS_ROCE_LDCP_##NAME##_OFS, \ + HNS_ROCE_LDCP_##NAME##_SZ, \ 0, HNS_ROCE_LDCP_##NAME##_MAX) HNS_PORT_LDCP_CC_ATTR_RW(cwd0, CWD0); @@ -264,7 +275,7 @@ static const struct attribute_group ldcp_cc_param_group = { }; #define HNS_PORT_HC3_CC_ATTR_RW(_name, NAME) \ - struct hns_port_cc_attr hns_roce_port_attr_hc3_##_name = \ + static struct hns_port_cc_attr hns_roce_port_attr_hc3_##_name = \ __HNS_SCC_ATTR(_name, HNS_ROCE_SCC_ALGO_HC3, \ HNS_ROCE_HC3_##NAME##_OFS, \ HNS_ROCE_HC3_##NAME##_SZ, \ @@ -298,7 +309,7 @@ static const struct attribute_group hc3_cc_param_group = { }; #define HNS_PORT_DIP_CC_ATTR_RW(_name, NAME) \ - struct hns_port_cc_attr hns_roce_port_attr_dip_##_name = \ + static struct hns_port_cc_attr hns_roce_port_attr_dip_##_name = \ __HNS_SCC_ATTR(_name, HNS_ROCE_SCC_ALGO_DIP, \ HNS_ROCE_DIP_##NAME##_OFS, \ HNS_ROCE_DIP_##NAME##_SZ, \