diff --git a/drivers/infiniband/hw/hns/Makefile b/drivers/infiniband/hw/hns/Makefile index 09f95fedd15db4c90a8b2fb8358e87b156f34306..04dce5ecbbc492280779d1da7feee8c5ad0aaf97 100644 --- a/drivers/infiniband/hw/hns/Makefile +++ b/drivers/infiniband/hw/hns/Makefile @@ -10,7 +10,8 @@ ccflags-y += -I $(srctree)/drivers/net/ethernet/hisilicon/hns3/hns3_common hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \ hns_roce_ah.o hns_roce_hem.o hns_roce_mr.o hns_roce_qp.o \ hns_roce_cq.o hns_roce_alloc.o hns_roce_db.o hns_roce_srq.o hns_roce_restrack.o \ - hns_roce_bond.o hns_roce_dca.o hns_roce_debugfs.o hns_roce_sysfs.o + hns_roce_bond.o hns_roce_dca.o hns_roce_debugfs.o hns_roce_sysfs.o \ + hns_roce_poe.o hns_roce_ext.o ifdef CONFIG_INFINIBAND_HNS_HIP08 hns-roce-hw-v2-objs := hns_roce_hw_v2.o $(hns-roce-objs) diff --git a/drivers/infiniband/hw/hns/hns_roce_common.h b/drivers/infiniband/hw/hns/hns_roce_common.h index 465d1f914b6c4c2faf66c48309a339f81ce47f5d..c2470b502b6818f1ba583e28e5b3864abcdc7ec9 100644 --- a/drivers/infiniband/hw/hns/hns_roce_common.h +++ b/drivers/infiniband/hw/hns/hns_roce_common.h @@ -42,6 +42,9 @@ #define roce_get_field(origin, mask, shift) \ ((le32_to_cpu(origin) & (mask)) >> (u32)(shift)) +#define roce_get_field64(origin, mask, shift) \ + ((le64_to_cpu(origin) & (mask)) >> (u32)(shift)) + #define roce_get_bit(origin, shift) \ roce_get_field((origin), (1ul << (shift)), (shift)) diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index 6997c3dbdc96db0e52ee8b287a29e0fc1f777a07..1858b0726c9f1b00102ac49913bf51724ea68d4e 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -311,10 +311,84 @@ static int get_cq_ucmd(struct hns_roce_cq *hr_cq, struct ib_udata *udata, return 0; } -static void set_cq_param(struct hns_roce_cq *hr_cq, u32 cq_entries, int vector, +static int set_poe_param(struct hns_roce_dev *hr_dev, + struct hns_roce_cq *hr_cq, + struct hns_roce_ib_create_cq *ucmd) +{ + if (!(ucmd->create_flags & HNS_ROCE_CREATE_CQ_FLAGS_POE_MODE)) + return 0; + + if (!poe_is_supported(hr_dev)) + return -EOPNOTSUPP; + + if (ucmd->poe_channel >= hr_dev->poe_ctx.poe_num) + return -EINVAL; + + if (!hr_dev->poe_ctx.poe_ch[ucmd->poe_channel].en) + return -EFAULT; + + hr_cq->flags |= HNS_ROCE_CQ_FLAG_POE_EN; + hr_cq->poe_channel = ucmd->poe_channel; + return 0; +} + +static bool is_notify_support(struct hns_roce_dev *hr_dev, + enum hns_roce_notify_mode notify_mode, + enum hns_roce_notify_device_en device_en) +{ + if (!is_write_notify_supported(hr_dev)) + return false; + + /* some configuration is not supported in HIP10 */ + if (hr_dev->pci_dev->revision != PCI_REVISION_ID_HIP10) + return true; + + if (notify_mode == HNS_ROCE_NOTIFY_MODE_64B_ALIGN || + device_en == HNS_ROCE_NOTIFY_DDR) { + ibdev_err(&hr_dev->ib_dev, "Unsupported notify_mode.\n"); + return false; + } + + return true; +} + +static int set_write_notify_param(struct hns_roce_dev *hr_dev, + struct hns_roce_cq *hr_cq, + struct hns_roce_ib_create_cq *ucmd) +{ +#define NOTIFY_MODE_MASK 0x3 + const struct { + u8 mode; + u8 mem_type; + } notify_attr[] = { + {HNS_ROCE_NOTIFY_MODE_64B_ALIGN, HNS_ROCE_NOTIFY_DEV}, + {HNS_ROCE_NOTIFY_MODE_4B_ALIGN, HNS_ROCE_NOTIFY_DEV}, + {HNS_ROCE_NOTIFY_MODE_64B_ALIGN, HNS_ROCE_NOTIFY_DDR}, + {HNS_ROCE_NOTIFY_MODE_4B_ALIGN, HNS_ROCE_NOTIFY_DDR}, + }; + u8 attr = ucmd->notify_mode & NOTIFY_MODE_MASK; + + if (!(ucmd->create_flags & HNS_ROCE_CREATE_CQ_FLAGS_WRITE_WITH_NOTIFY)) + return 0; + + if (!is_notify_support(hr_dev, notify_attr[attr].mode, + notify_attr[attr].mem_type)) + return -EOPNOTSUPP; + + hr_cq->flags |= HNS_ROCE_CQ_FLAG_NOTIFY_EN; + hr_cq->write_notify.notify_addr = + hr_dev->notify_tbl[ucmd->notify_idx].base_addr; + hr_cq->write_notify.notify_mode = notify_attr[attr].mode; + hr_cq->write_notify.notify_device_en = notify_attr[attr].mem_type; + + return 0; +} + +static int set_cq_param(struct hns_roce_cq *hr_cq, u32 cq_entries, int vector, struct hns_roce_ib_create_cq *ucmd) { struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device); + int ret; cq_entries = max(cq_entries, hr_dev->caps.min_cqes); cq_entries = roundup_pow_of_two(cq_entries); @@ -325,6 +399,19 @@ static void set_cq_param(struct hns_roce_cq *hr_cq, u32 cq_entries, int vector, spin_lock_init(&hr_cq->lock); INIT_LIST_HEAD(&hr_cq->sq_list); INIT_LIST_HEAD(&hr_cq->rq_list); + + if (!ucmd->create_flags) + return 0; + + if ((ucmd->create_flags & HNS_ROCE_CREATE_CQ_FLAGS_POE_MODE) && + (ucmd->create_flags & HNS_ROCE_CREATE_CQ_FLAGS_WRITE_WITH_NOTIFY)) + return -EINVAL; + + ret = set_poe_param(hr_dev, hr_cq, ucmd); + if (ret) + return ret; + + return set_write_notify_param(hr_dev, hr_cq, ucmd); } static int set_cqe_size(struct hns_roce_cq *hr_cq, struct ib_udata *udata, @@ -353,6 +440,22 @@ static int set_cqe_size(struct hns_roce_cq *hr_cq, struct ib_udata *udata, return 0; } +static void poe_ch_ref_cnt_inc(struct hns_roce_dev *hr_dev, + struct hns_roce_cq *hr_cq) +{ + struct hns_roce_poe_ch *poe_ch = + &hr_dev->poe_ctx.poe_ch[hr_cq->poe_channel]; + refcount_inc(&poe_ch->ref_cnt); +} + +static void poe_ch_ref_cnt_dec(struct hns_roce_dev *hr_dev, + struct hns_roce_cq *hr_cq) +{ + struct hns_roce_poe_ch *poe_ch = + &hr_dev->poe_ctx.poe_ch[hr_cq->poe_channel]; + refcount_dec(&poe_ch->ref_cnt); +} + int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr, struct ib_udata *udata) { @@ -379,7 +482,9 @@ int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr, } - set_cq_param(hr_cq, attr->cqe, attr->comp_vector, &ucmd); + ret = set_cq_param(hr_cq, attr->cqe, attr->comp_vector, &ucmd); + if (ret) + goto err_out; ret = set_cqe_size(hr_cq, udata, &ucmd); if (ret) @@ -412,6 +517,9 @@ int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr, if (udata) { resp.cqn = hr_cq->cqn; + resp.cap_flags = hr_cq->flags; + if (hr_cq->flags & HNS_ROCE_CQ_FLAG_POE_EN) + poe_ch_ref_cnt_inc(hr_dev, hr_cq); ret = ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp))); if (ret) @@ -444,6 +552,8 @@ int hns_roce_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device); struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq); + if (hr_cq->flags & HNS_ROCE_CQ_FLAG_POE_EN) + poe_ch_ref_cnt_dec(hr_dev, hr_cq); free_cqc(hr_dev, hr_cq); free_cqn(hr_dev, hr_cq->cqn); free_cq_db(hr_dev, hr_cq, udata); diff --git a/drivers/infiniband/hw/hns/hns_roce_debugfs.c b/drivers/infiniband/hw/hns/hns_roce_debugfs.c index 35d81e7546ed84189c1972a8dd92f0fd5f606efd..023b111997be5ee9a2e4b0934cbb7c1b3f988da2 100644 --- a/drivers/infiniband/hw/hns/hns_roce_debugfs.c +++ b/drivers/infiniband/hw/hns/hns_roce_debugfs.c @@ -74,10 +74,22 @@ struct hns_dca_debugfs { struct hns_dca_ctx_debugfs kctx; /* kDCA context */ }; +struct hns_poe_ch_debugfs { + struct dentry *root; /* dev debugfs entry */ + struct hns_debugfs_seqfile en; /* enable stats fir this ch */ + struct hns_debugfs_seqfile addr; /* addr of this ch */ + struct hns_debugfs_seqfile ref_cnt; /* ref_cnt for this ch */ +}; + +struct hns_poe_debugfs { + struct dentry *root; /* dev debugfs entry */ +}; + /* Debugfs for device */ struct hns_roce_dev_debugfs { struct dentry *root; struct hns_dca_debugfs *dca_root; + struct hns_poe_debugfs *poe_root; }; struct dca_mem_stats { @@ -497,6 +509,97 @@ static void destroy_dca_debugfs(struct hns_dca_debugfs *dca_dbgfs) kfree(dca_dbgfs); } +static int poe_debugfs_en_show(struct seq_file *file, void *offset) +{ + struct hns_roce_poe_ch *poe_ch = file->private; + + seq_printf(file, "%-10s\n", poe_ch->en ? "enable" : "disable"); + return 0; +} + +static int poe_debugfs_addr_show(struct seq_file *file, void *offset) +{ +#define POE_ADDR_OFFSET_MASK GENMASK(31, 0) + struct hns_roce_poe_ch *poe_ch = file->private; + + seq_printf(file, "0x%llx\n", poe_ch->addr & POE_ADDR_OFFSET_MASK); + return 0; +} + +static int poe_debugfs_ref_cnt_show(struct seq_file *file, void *offset) +{ + struct hns_roce_poe_ch *poe_ch = file->private; + + seq_printf(file, "0x%-10u\n", refcount_read(&poe_ch->ref_cnt)); + return 0; +} + +static void init_poe_ch_debugfs(struct hns_roce_dev *hr_dev, uint8_t index, + struct dentry *parent) +{ +#define POE_CH_NAME_LEN 10 + struct hns_roce_poe_ch *poe_ch = &hr_dev->poe_ctx.poe_ch[index]; + struct hns_poe_ch_debugfs *dbgfs; + char name[POE_CH_NAME_LEN]; + + dbgfs = kvzalloc(sizeof(*dbgfs), GFP_KERNEL); + if (!dbgfs) + return; + + snprintf(name, sizeof(name), "poe_%u", index); + dbgfs->root = debugfs_create_dir(name, parent); + + init_debugfs_seqfile(&dbgfs->en, "en", dbgfs->root, + poe_debugfs_en_show, poe_ch); + init_debugfs_seqfile(&dbgfs->addr, "addr", dbgfs->root, + poe_debugfs_addr_show, poe_ch); + init_debugfs_seqfile(&dbgfs->ref_cnt, "ref_cnt", dbgfs->root, + poe_debugfs_ref_cnt_show, poe_ch); + poe_ch->poe_ch_debugfs = dbgfs; +} + +static void cleanup_poe_ch_debugfs(struct hns_roce_dev *hr_dev, uint8_t index) +{ + struct hns_roce_poe_ch *poe_ch = &hr_dev->poe_ctx.poe_ch[index]; + struct hns_poe_ch_debugfs *dbgfs = poe_ch->poe_ch_debugfs; + + cleanup_debugfs_seqfile(&dbgfs->en); + cleanup_debugfs_seqfile(&dbgfs->addr); + cleanup_debugfs_seqfile(&dbgfs->ref_cnt); + debugfs_remove_recursive(dbgfs->root); + kvfree(dbgfs); +} + +static struct hns_poe_debugfs * +create_poe_debugfs(struct hns_roce_dev *hr_dev, struct dentry *parent) +{ + struct hns_poe_debugfs *dbgfs; + int i; + + dbgfs = kvzalloc(sizeof(*dbgfs), GFP_KERNEL); + if (!dbgfs) + return NULL; + + dbgfs->root = debugfs_create_dir("poe", parent); + + for (i = 0; i < hr_dev->poe_ctx.poe_num; i++) + init_poe_ch_debugfs(hr_dev, i, dbgfs->root); + + return dbgfs; +} + +static void destroy_poe_debugfs(struct hns_roce_dev *hr_dev, + struct hns_poe_debugfs *poe_dbgfs) +{ + int i; + + for (i = 0; i < hr_dev->poe_ctx.poe_num; i++) + cleanup_poe_ch_debugfs(hr_dev, i); + + debugfs_remove_recursive(poe_dbgfs->root); + kvfree(poe_dbgfs); +} + /* debugfs for ucontext */ void hns_roce_register_uctx_debugfs(struct hns_roce_dev *hr_dev, struct hns_roce_ucontext *uctx) @@ -553,6 +656,9 @@ void hns_roce_register_debugfs(struct hns_roce_dev *hr_dev) if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_DCA_MODE) dbgfs->dca_root = create_dca_debugfs(hr_dev, dbgfs->root); + if (poe_is_supported(hr_dev)) + dbgfs->poe_root = create_poe_debugfs(hr_dev, dbgfs->root); + hr_dev->dbgfs = dbgfs; } @@ -572,6 +678,11 @@ void hns_roce_unregister_debugfs(struct hns_roce_dev *hr_dev) dbgfs->dca_root = NULL; } + if (dbgfs->poe_root) { + destroy_poe_debugfs(hr_dev, dbgfs->poe_root); + dbgfs->poe_root = NULL; + } + debugfs_remove_recursive(dbgfs->root); hr_dev->dbgfs = NULL; kfree(dbgfs); diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 14bc242b6ad6c6524dee14e5a88374feaafc27dc..d376e23f2a761061e0d75827cf557f9a0594339e 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -37,9 +37,11 @@ #include #include #include "hns_roce_bond.h" +#include "hns_roce_ext.h" #define PCI_REVISION_ID_HIP08 0x21 #define PCI_REVISION_ID_HIP09 0x30 +#define PCI_REVISION_ID_HIP10 0x32 #define HNS_ROCE_MAX_MSG_LEN 0x80000000 @@ -104,6 +106,10 @@ #define CQ_BANKID_SHIFT 2 #define CQ_BANKID_MASK GENMASK(1, 0) +#define MAX_NOTIFY_MEM_SIZE BIT(24) + +#define HNS_ROCE_MEM_BAR 2 + enum { SERV_TYPE_RC, SERV_TYPE_UC, @@ -160,10 +166,12 @@ enum { HNS_ROCE_CAP_FLAG_SVE_DIRECT_WQE = BIT(13), HNS_ROCE_CAP_FLAG_SDI_MODE = BIT(14), HNS_ROCE_CAP_FLAG_DCA_MODE = BIT(15), + HNS_ROCE_CAP_FLAG_WRITE_NOTIFY = BIT(16), HNS_ROCE_CAP_FLAG_STASH = BIT(17), HNS_ROCE_CAP_FLAG_CQE_INLINE = BIT(19), HNS_ROCE_CAP_FLAG_BOND = BIT(21), HNS_ROCE_CAP_FLAG_SRQ_RECORD_DB = BIT(22), + HNS_ROCE_CAP_FLAG_POE = BIT(27), }; #define HNS_ROCE_DB_TYPE_COUNT 2 @@ -221,6 +229,18 @@ struct hns_user_mmap_entry { u64 address; }; +struct hns_roce_poe_ch { + uint8_t en; + refcount_t ref_cnt; + uint64_t addr; + void *poe_ch_debugfs; +}; + +struct hns_roce_poe_ctx { + uint8_t poe_num; + struct hns_roce_poe_ch *poe_ch; +}; + struct hns_roce_dca_ctx { struct list_head pool; /* all DCA mems link to @pool */ spinlock_t pool_lock; /* protect @pool */ @@ -473,6 +493,22 @@ struct hns_roce_db { unsigned long order; }; +enum hns_roce_notify_mode { + HNS_ROCE_NOTIFY_MODE_64B_ALIGN = 0, + HNS_ROCE_NOTIFY_MODE_4B_ALIGN = 1, +}; + +enum hns_roce_notify_device_en { + HNS_ROCE_NOTIFY_DEV = 0, + HNS_ROCE_NOTIFY_DDR = 1, +}; + +struct hns_roce_notify_conf { + u64 notify_addr; /* should be aligned to 4k */ + u8 notify_mode; /* use enum hns_roce_notify_mode */ + u8 notify_device_en; /* use enum hns_roce_notify_device_en */ +}; + struct hns_roce_cq { struct ib_cq ib_cq; struct hns_roce_mtr mtr; @@ -493,6 +529,8 @@ struct hns_roce_cq { struct list_head rq_list; /* all qps on this recv cq */ int is_armed; /* cq is armed */ struct list_head node; /* all armed cqs are on a list */ + u8 poe_channel; + struct hns_roce_notify_conf write_notify; }; struct hns_roce_idx_que { @@ -788,6 +826,8 @@ enum congest_type { HNS_ROCE_CONGEST_TYPE_DIP = 1 << HNS_ROCE_SCC_ALGO_DIP, }; +#define HNS_ROCE_POE_CH_NUM 4 + struct hns_roce_caps { u64 fw_ver; u8 num_ports; @@ -919,6 +959,7 @@ struct hns_roce_caps { u16 default_ceq_arm_st; u8 congest_type; u8 default_congest_type; + u8 poe_ch_num; }; enum hns_roce_device_state { @@ -1034,6 +1075,7 @@ struct hns_roce_hw { enum hns_roce_scc_algo algo); int (*query_scc_param)(struct hns_roce_dev *hr_dev, u8 port_num, enum hns_roce_scc_algo alog); + int (*cfg_poe_ch)(struct hns_roce_dev *hr_dev, u32 index, u64 poe_addr); }; #define HNS_ROCE_SCC_PARAM_SIZE 4 @@ -1092,6 +1134,10 @@ struct hns_roce_dev { u32 vendor_id; u32 vendor_part_id; u32 hw_rev; + u16 chip_id; + u16 die_id; + u16 mac_id; + u16 func_id; void __iomem *priv_addr; struct hns_roce_cmdq cmd; @@ -1129,6 +1175,10 @@ struct hns_roce_dev { struct notifier_block bond_nb; struct hns_roce_port port_data[HNS_ROCE_MAX_PORTS]; atomic64_t *dfx_cnt; + struct hns_roce_poe_ctx poe_ctx; /* poe ch array */ + + struct rdma_notify_mem *notify_tbl; + size_t notify_num; }; static inline struct hns_roce_dev *to_hr_dev(struct ib_device *ib_dev) @@ -1286,6 +1336,16 @@ static inline u8 get_hr_bus_num(struct hns_roce_dev *hr_dev) return hr_dev->pci_dev->bus->number; } +static inline bool poe_is_supported(struct hns_roce_dev *hr_dev) +{ + return !!(hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_POE); +} + +static inline bool is_write_notify_supported(struct hns_roce_dev *dev) +{ + return !!(dev->caps.flags & HNS_ROCE_CAP_FLAG_WRITE_NOTIFY); +} + void hns_roce_init_uar_table(struct hns_roce_dev *dev); int hns_roce_uar_alloc(struct hns_roce_dev *dev, struct hns_roce_uar *uar); @@ -1427,4 +1487,7 @@ hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address, int hns_roce_create_port_files(struct ib_device *ibdev, u8 port_num, struct kobject *kobj); void hns_roce_unregister_sysfs(struct hns_roce_dev *hr_dev); +int hns_roce_register_poe_channel(struct hns_roce_dev *hr_dev, u8 channel, + u64 poe_addr); +int hns_roce_unregister_poe_channel(struct hns_roce_dev *hr_dev, u8 channel); #endif /* _HNS_ROCE_DEVICE_H */ diff --git a/drivers/infiniband/hw/hns/hns_roce_ext.c b/drivers/infiniband/hw/hns/hns_roce_ext.c new file mode 100644 index 0000000000000000000000000000000000000000..63d0a48abe71e81ce456ac345061820e93a1c4bf --- /dev/null +++ b/drivers/infiniband/hw/hns/hns_roce_ext.c @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* + * Copyright (c) 2023 Hisilicon Limited. + */ + +#include +#include "hns_roce_device.h" + +static bool is_hns_roce(struct ib_device *ib_dev) +{ + if (ib_dev && ib_dev->ops.driver_id == RDMA_DRIVER_HNS) + return true; + + return false; +} + +static bool is_hns_roce_vf(struct hns_roce_dev *hr_dev) +{ + return hr_dev->is_vf; +} + +bool rdma_support_stars(struct ib_device *ib_dev) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev); + + if (!is_hns_roce(ib_dev) || is_hns_roce_vf(hr_dev)) + return false; + + if (poe_is_supported(hr_dev) && is_write_notify_supported(hr_dev)) + return true; + + return false; +} +EXPORT_SYMBOL(rdma_support_stars); + +int rdma_register_poe_channel(struct ib_device *ib_dev, u8 channel, + u64 poe_addr) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev); + + if (!is_hns_roce(ib_dev)) + return -EOPNOTSUPP; + + return hns_roce_register_poe_channel(hr_dev, channel, poe_addr); +} +EXPORT_SYMBOL(rdma_register_poe_channel); + +int rdma_unregister_poe_channel(struct ib_device *ib_dev, u8 channel) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev); + + if (!is_hns_roce(ib_dev)) + return -EOPNOTSUPP; + + return hns_roce_unregister_poe_channel(hr_dev, channel); +} +EXPORT_SYMBOL(rdma_unregister_poe_channel); + +u64 rdma_query_qp_db(struct ib_device *ib_dev, int qp_index) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev); + u64 bar_addr; + + if (!rdma_support_stars(ib_dev)) + return 0; + + bar_addr = pci_resource_start(hr_dev->pci_dev, HNS_ROCE_MEM_BAR); + return bar_addr + hr_dev->sdb_offset + + DB_REG_OFFSET * hr_dev->priv_uar.index; +} +EXPORT_SYMBOL(rdma_query_qp_db); + +int rdma_query_hw_id(struct ib_device *ib_dev, u32 *chip_id, + u32 *die_id, u32 *func_id) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev); + + if (!is_hns_roce(ib_dev) || is_hns_roce_vf(hr_dev)) + return -EOPNOTSUPP; + + if (!chip_id || !die_id || !func_id) + return -EINVAL; + + if (hr_dev->chip_id == HNS_IB_INVALID_ID) + return -EINVAL; + + *chip_id = hr_dev->chip_id; + *die_id = hr_dev->die_id; + *func_id = hr_dev->func_id; + return 0; +} +EXPORT_SYMBOL(rdma_query_hw_id); + +int rdma_register_notify_addr(struct ib_device *ib_dev, + size_t num, struct rdma_notify_mem *notify_mem) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev); + size_t i; + + if (!is_hns_roce(ib_dev) || !is_write_notify_supported(hr_dev)) + return -EOPNOTSUPP; + + if (hr_dev->notify_tbl) + return -EBUSY; + + if (!num || !notify_mem) + return -EINVAL; + + for (i = 0; i < num; i++) { + if (!notify_mem[i].size || + notify_mem[i].size > MAX_NOTIFY_MEM_SIZE) + return -EINVAL; + if (!notify_mem[i].base_addr) + return -EINVAL; + } + + hr_dev->notify_tbl = kvmalloc_array(num, sizeof(*notify_mem), + GFP_KERNEL); + if (!hr_dev->notify_tbl) + return -ENOMEM; + + hr_dev->notify_num = num; + memcpy(hr_dev->notify_tbl, notify_mem, sizeof(*notify_mem) * num); + + return 0; +} +EXPORT_SYMBOL(rdma_register_notify_addr); + +int rdma_unregister_notify_addr(struct ib_device *ib_dev) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev); + + if (!is_hns_roce(ib_dev) || !is_write_notify_supported(hr_dev)) + return -EOPNOTSUPP; + + if (hr_dev->notify_tbl) + kvfree(hr_dev->notify_tbl); + + hr_dev->notify_tbl = NULL; + + return 0; +} +EXPORT_SYMBOL(rdma_unregister_notify_addr); diff --git a/drivers/infiniband/hw/hns/hns_roce_ext.h b/drivers/infiniband/hw/hns/hns_roce_ext.h new file mode 100644 index 0000000000000000000000000000000000000000..603e51cbf29a6f903e98867f320114ac2fe88107 --- /dev/null +++ b/drivers/infiniband/hw/hns/hns_roce_ext.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB + * + * Copyright (c) 2023 Hisilicon Limited. + */ + +#ifndef __HNS_ROCE_EXT_H +#define __HNS_ROCE_EXT_H +#include + +/** + * rdma_register_notify_addr - Register an POE channel for this RDMA device. + * @channel - POE channel index. + * @poe_addr - POE channel address. + * + * If the current POE device is not associated with CQ, then it will be + * allowed to be re-registered. Otherwise, re-registration or + * de-registration will report an EBUSY error. + */ +int rdma_register_poe_channel(struct ib_device *ib_dev, u8 channel, u64 poe_addr); +int rdma_unregister_poe_channel(struct ib_device *ib_dev, u8 channel); + +/** + * rdma_support_stars - Helper function to determine whether the + * current device supports STARS. + */ +bool rdma_support_stars(struct ib_device *ib_dev); + +/** + * rdma_query_qp_db - Helper function to get the doorbell address of this + * device. Currently, it only supports use in STARS scenarios. + * @qp_index - QP number. + */ +u64 rdma_query_qp_db(struct ib_device *ib_dev, int qp_index); + +/** + * rdma_query_hw_id - Get the relevant hardware ID of the current device. + * @chip_id - The ID of the chip where the current device is located. + * @die_id - The ID of the IO DIE where the current device is located. + * @func_id - The function ID of this device. + */ +int rdma_query_hw_id(struct ib_device *ib_dev, u32 *chip_id, + u32 *die_id, u32 *func_id); +/** + * struct rdma_notify_mem + * @base_addr - The memory region base addr for write with notify operation. + * @size - size of the notify memory region + */ +struct rdma_notify_mem { + u64 base_addr; + u32 size; +}; + +/** + * rdma_register_notify_addr - Register an memory region which will be used by + * write with notify operation. + * @num - How many elements in array + * @notify_mem - Notify memory array. + * + * If notify_mem has already been registered, re-registration + * will not be allowed. + */ +int rdma_register_notify_addr(struct ib_device *ib_dev, + size_t num, struct rdma_notify_mem *notify_mem); +int rdma_unregister_notify_addr(struct ib_device *ib_dev); + +#endif diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index a9b44ebbbd259e7e82520262814f461b42df0ce6..9f1612d5747ffb83a2a181419b41e102efb4c988 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -1436,9 +1436,11 @@ static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev, if (likely(desc_ret == CMD_EXEC_SUCCESS)) continue; - dev_err_ratelimited(hr_dev->dev, - "Cmdq IO error, opcode = 0x%x, return = 0x%x.\n", - desc->opcode, desc_ret); + if (desc->opcode != HNS_ROCE_OPC_QUERY_HW_ID && + desc_ret != CMD_NOT_EXIST) + dev_err_ratelimited(hr_dev->dev, + "Cmdq IO error, opcode = 0x%x, return = 0x%x.\n", + desc->opcode, desc_ret); ret = hns_roce_cmd_err_convert_errno(desc_ret); } } else { @@ -1570,6 +1572,39 @@ static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev) return 0; } +static void hns_roce_cmq_query_hw_id(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_hw_id_query_cmq *resp; + struct hns_roce_cmq_desc desc; + int ret; + + if (hr_dev->is_vf) + goto invalid_val; + + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_HW_ID, true); + ret = hns_roce_cmq_send(hr_dev, &desc, 1); + if (ret) { + if (desc.retval != CMD_NOT_EXIST) + ibdev_warn(&hr_dev->ib_dev, + "failed to query hw id, ret = %d.\n", ret); + + goto invalid_val; + } + + resp = (struct hns_roce_hw_id_query_cmq *)desc.data; + hr_dev->chip_id = resp->chip_id; + hr_dev->die_id = resp->die_id; + hr_dev->mac_id = resp->mac_id; + hr_dev->func_id = (u16)le32_to_cpu(resp->func_id); + return; + +invalid_val: + hr_dev->func_id = HNS_IB_INVALID_ID; + hr_dev->mac_id = HNS_IB_INVALID_ID; + hr_dev->die_id = HNS_IB_INVALID_ID; + hr_dev->chip_id = HNS_IB_INVALID_ID; +} + static void func_clr_hw_resetting_state(struct hns_roce_dev *hr_dev, struct hnae3_handle *handle) { @@ -2413,6 +2448,9 @@ static int hns_roce_query_caps(struct hns_roce_dev *hr_dev) if (!(caps->page_size_cap & PAGE_SIZE)) caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED; + if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) + caps->poe_ch_num = HNS_ROCE_POE_CH_NUM; + if (!hr_dev->is_vf) { caps->cqe_sz = resp_a->cqe_sz; caps->qpc_sz = le16_to_cpu(resp_b->qpc_sz); @@ -2586,6 +2624,8 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) return ret; } + hns_roce_cmq_query_hw_id(hr_dev); + hr_dev->vendor_part_id = hr_dev->pci_dev->device; hr_dev->sys_image_guid = be64_to_cpu(hr_dev->ib_dev.node_guid); @@ -3810,6 +3850,44 @@ static void hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn, spin_unlock_irq(&hr_cq->lock); } +static void enable_write_notify(struct hns_roce_cq *hr_cq, + struct hns_roce_v2_cq_context *cq_context) +{ + hr_reg_enable(cq_context, CQC_NOTIFY_EN); + hr_reg_write(cq_context, CQC_NOTIFY_DEVICE_EN, + hr_cq->write_notify.notify_device_en); + hr_reg_write(cq_context, CQC_NOTIFY_MODE, + hr_cq->write_notify.notify_mode); + hr_reg_write(cq_context, CQC_NOTIFY_ADDR_0, + (u32)roce_get_field64(hr_cq->write_notify.notify_addr, + CQC_NOTIFY_ADDR_0_M, + CQC_NOTIFY_ADDR_0_S)); + hr_reg_write(cq_context, CQC_NOTIFY_ADDR_1, + (u32)roce_get_field64(hr_cq->write_notify.notify_addr, + CQC_NOTIFY_ADDR_1_M, + CQC_NOTIFY_ADDR_1_S)); + hr_reg_write(cq_context, CQC_NOTIFY_ADDR_2, + (u32)roce_get_field64(hr_cq->write_notify.notify_addr, + CQC_NOTIFY_ADDR_2_M, + CQC_NOTIFY_ADDR_2_S)); + hr_reg_write(cq_context, CQC_NOTIFY_ADDR_3, + (u32)roce_get_field64(hr_cq->write_notify.notify_addr, + CQC_NOTIFY_ADDR_3_M, + CQC_NOTIFY_ADDR_3_S)); + hr_reg_write(cq_context, CQC_NOTIFY_ADDR_4, + (u32)roce_get_field64(hr_cq->write_notify.notify_addr, + CQC_NOTIFY_ADDR_4_M, + CQC_NOTIFY_ADDR_4_S)); + hr_reg_write(cq_context, CQC_NOTIFY_ADDR_5, + (u32)roce_get_field64(hr_cq->write_notify.notify_addr, + CQC_NOTIFY_ADDR_5_M, + CQC_NOTIFY_ADDR_5_S)); + hr_reg_write(cq_context, CQC_NOTIFY_ADDR_6, + (u32)roce_get_field64(hr_cq->write_notify.notify_addr, + CQC_NOTIFY_ADDR_6_M, + CQC_NOTIFY_ADDR_6_S)); +} + static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, void *mb_buf, u64 *mtts, dma_addr_t dma_handle) @@ -3825,12 +3903,19 @@ static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev, hr_reg_write(cq_context, CQC_CEQN, hr_cq->vector); hr_reg_write(cq_context, CQC_CQN, hr_cq->cqn); - if (hr_cq->cqe_size == HNS_ROCE_V3_CQE_SIZE) - hr_reg_write(cq_context, CQC_CQE_SIZE, CQE_SIZE_64B); + if (hr_cq->flags & HNS_ROCE_CQ_FLAG_POE_EN) { + hr_reg_enable(cq_context, CQC_POE_EN); + hr_reg_write(cq_context, CQC_POE_NUM, hr_cq->poe_channel); + } - if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_STASH) + if (hr_cq->flags & HNS_ROCE_CQ_FLAG_NOTIFY_EN) + enable_write_notify(hr_cq, cq_context); + else if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_STASH) hr_reg_enable(cq_context, CQC_STASH); + if (hr_cq->cqe_size == HNS_ROCE_V3_CQE_SIZE) + hr_reg_write(cq_context, CQC_CQE_SIZE, CQE_SIZE_64B); + hr_reg_write(cq_context, CQC_CQE_CUR_BLK_ADDR_L, to_hr_hw_page_addr(mtts[0])); hr_reg_write(cq_context, CQC_CQE_CUR_BLK_ADDR_H, @@ -4496,6 +4581,12 @@ static void set_access_flags(struct hns_roce_qp *hr_qp, hr_reg_write_bool(context, QPC_EXT_ATE, access_flags & IB_ACCESS_REMOTE_ATOMIC); hr_reg_clear(qpc_mask, QPC_EXT_ATE); + + if ((hr_qp->en_flags & HNS_ROCE_QP_CAP_WRITE_WITH_NOTIFY) && + (access_flags & IB_ACCESS_REMOTE_WRITE)) { + hr_reg_enable(context, QPC_WN_EN); + hr_reg_clear(qpc_mask, QPC_WN_EN); + } } static void set_qpc_wqe_cnt(struct hns_roce_qp *hr_qp, @@ -4578,7 +4669,8 @@ static void modify_qp_reset_to_init(struct ib_qp *ibqp, if (hr_dev->caps.qpc_sz < HNS_ROCE_V3_QPC_SZ) return; - if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_STASH) + if (!(hr_qp->en_flags & HNS_ROCE_QP_CAP_WRITE_WITH_NOTIFY) && + (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_STASH)) hr_reg_enable(&context->ext, QPCEX_STASH); } @@ -7141,6 +7233,68 @@ static int hns_roce_v2_query_scc_param(struct hns_roce_dev *hr_dev, return 0; } +static int config_poe_addr(struct hns_roce_dev *hr_dev, + u32 channel_id, u64 addr) +{ + struct hns_roce_poe_cfg_addr_cmq *cmd; + struct hns_roce_cmq_desc desc; + int ret; + + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_POE_ADDR, false); + cmd = (struct hns_roce_poe_cfg_addr_cmq *)desc.data; + cmd->channel_id = cpu_to_le32(channel_id); + cmd->poe_addr_l = cpu_to_le32(lower_32_bits(addr)); + cmd->poe_addr_h = cpu_to_le32(upper_32_bits(addr)); + + ret = hns_roce_cmq_send(hr_dev, &desc, 1); + if (ret) + ibdev_err_ratelimited(&hr_dev->ib_dev, + "configure poe channel %u addr failed, ret = %d.\n", + channel_id, ret); + return ret; +} + +static int config_poe_attr(struct hns_roce_dev *hr_dev, u32 channel_id, bool en) +{ + struct hns_roce_poe_cfg_attr_cmq *cmd; + struct hns_roce_cmq_desc desc; + int ret; + + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_POE_ATTR, false); + cmd = (struct hns_roce_poe_cfg_attr_cmq *)desc.data; + cmd->channel_id = cpu_to_le32(channel_id); + cmd->rsv_en_outstd = en ? 1 : 0; + + ret = hns_roce_cmq_send(hr_dev, &desc, 1); + if (ret) + ibdev_err_ratelimited(&hr_dev->ib_dev, + "configure poe channel %u attr failed, ret = %d.\n", + channel_id, ret); + return ret; +} + +static int hns_roce_cfg_poe_ch(struct hns_roce_dev *hr_dev, u32 index, + u64 poe_addr) +{ + int ret; + + if (index >= hr_dev->caps.poe_ch_num) { + ibdev_err_ratelimited(&hr_dev->ib_dev, + "invalid POE channel %u.\n", index); + return -EINVAL; + } + + ret = config_poe_addr(hr_dev, index, poe_addr); + if (ret) + return ret; + + ret = config_poe_attr(hr_dev, index, !!poe_addr); + if (ret) + config_poe_addr(hr_dev, index, 0); + + return ret; +} + static const struct ib_device_ops hns_roce_v2_dev_ops = { .destroy_qp = hns_roce_v2_destroy_qp, .modify_cq = hns_roce_v2_modify_cq, @@ -7196,6 +7350,7 @@ static const struct hns_roce_hw hns_roce_hw_v2 = { .query_hw_counter = hns_roce_hw_v2_query_counter, .config_scc_param = hns_roce_v2_config_scc_param, .query_scc_param = hns_roce_v2_query_scc_param, + .cfg_poe_ch = hns_roce_cfg_poe_ch, }; static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = { diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h index 4293ec92ca534c3c7196ac4a95bea3ef6a517711..01a70ea86af18f80971288f96d2d87cb9c93b277 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -200,9 +200,12 @@ enum hns_roce_opcode_type { HNS_ROCE_OPC_CFG_LDCP_PARAM = 0x1A81, HNS_ROCE_OPC_CFG_HC3_PARAM = 0x1A82, HNS_ROCE_OPC_CFG_DIP_PARAM = 0x1A83, + HNS_ROCE_OPC_QUERY_HW_ID = 0x7032, HNS_ROCE_OPC_QUERY_HW_VER = 0x8000, HNS_ROCE_OPC_CFG_GLOBAL_PARAM = 0x8001, HNS_ROCE_OPC_ALLOC_PF_RES = 0x8004, + HNS_ROCE_OPC_CFG_POE_ADDR = 0x801B, + HNS_ROCE_OPC_CFG_POE_ATTR = 0x801C, HNS_ROCE_OPC_QUERY_COUNTER = 0x8206, HNS_ROCE_OPC_QUERY_PF_RES = 0x8400, HNS_ROCE_OPC_ALLOC_VF_RES = 0x8401, @@ -288,32 +291,54 @@ struct hns_roce_v2_cq_context { #define HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM 0x0 #define HNS_ROCE_V2_CQ_DEFAULT_INTERVAL 0x0 +#define CQC_NOTIFY_ADDR_0_S 12 +#define CQC_NOTIFY_ADDR_0_M GENMASK(19, 12) +#define CQC_NOTIFY_ADDR_1_S 20 +#define CQC_NOTIFY_ADDR_1_M GENMASK(29, 20) +#define CQC_NOTIFY_ADDR_2_S 30 +#define CQC_NOTIFY_ADDR_2_M GENMASK(33, 30) +#define CQC_NOTIFY_ADDR_3_S 34 +#define CQC_NOTIFY_ADDR_3_M GENMASK(41, 34) +#define CQC_NOTIFY_ADDR_4_S 42 +#define CQC_NOTIFY_ADDR_4_M GENMASK(49, 42) +#define CQC_NOTIFY_ADDR_5_S 50 +#define CQC_NOTIFY_ADDR_5_M GENMASK(57, 50) +#define CQC_NOTIFY_ADDR_6_S 58 +#define CQC_NOTIFY_ADDR_6_M GENMASK(63, 58) + #define CQC_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_v2_cq_context, h, l) #define CQC_CQ_ST CQC_FIELD_LOC(1, 0) #define CQC_POLL CQC_FIELD_LOC(2, 2) #define CQC_SE CQC_FIELD_LOC(3, 3) #define CQC_OVER_IGNORE CQC_FIELD_LOC(4, 4) +#define CQC_NOTIFY_MODE CQC_FIELD_LOC(4, 4) #define CQC_ARM_ST CQC_FIELD_LOC(7, 6) #define CQC_SHIFT CQC_FIELD_LOC(12, 8) #define CQC_CMD_SN CQC_FIELD_LOC(14, 13) #define CQC_CEQN CQC_FIELD_LOC(23, 15) +#define CQC_NOTIFY_ADDR_0 CQC_FIELD_LOC(31, 24) #define CQC_CQN CQC_FIELD_LOC(55, 32) #define CQC_POE_EN CQC_FIELD_LOC(56, 56) #define CQC_POE_NUM CQC_FIELD_LOC(58, 57) #define CQC_CQE_SIZE CQC_FIELD_LOC(60, 59) #define CQC_CQ_CNT_MODE CQC_FIELD_LOC(61, 61) +#define CQC_NOTIFY_DEVICE_EN CQC_FIELD_LOC(62, 62) #define CQC_STASH CQC_FIELD_LOC(63, 63) #define CQC_CQE_CUR_BLK_ADDR_L CQC_FIELD_LOC(95, 64) #define CQC_CQE_CUR_BLK_ADDR_H CQC_FIELD_LOC(115, 96) #define CQC_POE_QID CQC_FIELD_LOC(125, 116) +#define CQC_NOTIFY_ADDR_1 CQC_FIELD_LOC(125, 116) #define CQC_CQE_HOP_NUM CQC_FIELD_LOC(127, 126) #define CQC_CQE_NEX_BLK_ADDR_L CQC_FIELD_LOC(159, 128) #define CQC_CQE_NEX_BLK_ADDR_H CQC_FIELD_LOC(179, 160) +#define CQC_NOTIFY_ADDR_2 CQC_FIELD_LOC(183, 180) #define CQC_CQE_BAR_PG_SZ CQC_FIELD_LOC(187, 184) #define CQC_CQE_BUF_PG_SZ CQC_FIELD_LOC(191, 188) #define CQC_CQ_PRODUCER_IDX CQC_FIELD_LOC(215, 192) +#define CQC_NOTIFY_ADDR_3 CQC_FIELD_LOC(223, 216) #define CQC_CQ_CONSUMER_IDX CQC_FIELD_LOC(247, 224) +#define CQC_NOTIFY_ADDR_4 CQC_FIELD_LOC(255, 248) #define CQC_CQE_BA_L CQC_FIELD_LOC(287, 256) #define CQC_CQE_BA_H CQC_FIELD_LOC(316, 288) #define CQC_POE_QID_H_0 CQC_FIELD_LOC(319, 317) @@ -321,11 +346,14 @@ struct hns_roce_v2_cq_context { #define CQC_CQE_DB_RECORD_ADDR_L CQC_FIELD_LOC(351, 321) #define CQC_CQE_DB_RECORD_ADDR_H CQC_FIELD_LOC(383, 352) #define CQC_CQE_CNT CQC_FIELD_LOC(407, 384) +#define CQC_NOTIFY_ADDR_5 CQC_FIELD_LOC(415, 408) #define CQC_CQ_MAX_CNT CQC_FIELD_LOC(431, 416) #define CQC_CQ_PERIOD CQC_FIELD_LOC(447, 432) #define CQC_CQE_REPORT_TIMER CQC_FIELD_LOC(471, 448) #define CQC_WR_CQE_IDX CQC_FIELD_LOC(479, 472) #define CQC_SE_CQE_IDX CQC_FIELD_LOC(503, 480) +#define CQC_NOTIFY_ADDR_6 CQC_FIELD_LOC(509, 504) +#define CQC_NOTIFY_EN CQC_FIELD_LOC(510, 510) #define CQC_POE_QID_H_1 CQC_FIELD_LOC(511, 511) struct hns_roce_srq_context { @@ -945,6 +973,15 @@ struct hns_roce_v2_wqe_data_seg { __le64 addr; }; +struct hns_roce_hw_id_query_cmq { + __u8 chip_id; + __u8 die_id; + __u8 mac_id; + __u8 reserved; + __le32 func_id; + __le32 rsv[4]; +}; + struct hns_roce_query_version { __le16 rocee_vendor_id; __le16 rocee_hw_version; @@ -1278,6 +1315,21 @@ struct hns_roce_cmq_req { __le32 data[6]; }; +struct hns_roce_poe_cfg_addr_cmq { + __le32 channel_id; + __le32 poe_addr_l; + __le32 poe_addr_h; + __le32 rsv[3]; +}; + +#define V2_POE_ATTR_EN V2_POE_ATTR_FIELD_LOC(40, 40) +#define V2_POE_ATTR_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_poe_cfg_attr_cmq, h, l) +struct hns_roce_poe_cfg_attr_cmq { + __le32 channel_id; + __le32 rsv_en_outstd; + __le32 rsv[4]; +}; + #define CMQ_REQ_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_cmq_req, h, l) struct hns_roce_cmq_desc { diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index 51628ab5bae4a07b6dc4a3ad3dcc551e770b1efc..239e08abd166a20016afdb96b0c6c5fdfd73d198 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -234,6 +234,32 @@ static int hns_roce_setup_mtu_mac(struct hns_roce_dev *hr_dev) return 0; } +static int set_attrx(struct hns_roce_dev *hr_dev, struct ib_udata *uhw) +{ + struct hns_roce_ib_query_device_resp resp = {}; + size_t uhw_outlen; + + if (!uhw || !uhw->outlen) + return 0; + + uhw_outlen = uhw->outlen; + resp.len = sizeof(resp.comp_mask) + sizeof(resp.len); + if (uhw_outlen < resp.len) + return -EINVAL; + + if (uhw->inlen && !ib_is_udata_cleared(uhw, 0, uhw->inlen)) + return -EINVAL; + + if (uhw_outlen >= offsetofend(typeof(resp), hw_id)) { + resp.len += sizeof(resp.hw_id); + resp.hw_id.chip_id = hr_dev->chip_id; + resp.hw_id.die_id = hr_dev->die_id; + resp.hw_id.func_id = hr_dev->func_id; + } + + return ib_copy_to_udata(uhw, &resp, resp.len); +} + static int hns_roce_query_device(struct ib_device *ib_dev, struct ib_device_attr *props, struct ib_udata *uhw) @@ -281,7 +307,7 @@ static int hns_roce_query_device(struct ib_device *ib_dev, if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC) props->device_cap_flags |= IB_DEVICE_XRC; - return 0; + return set_attrx(hr_dev, uhw); } static int hns_roce_query_port(struct ib_device *ib_dev, u8 port_num, @@ -1401,6 +1427,36 @@ void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev) spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags); } +static void hns_roce_register_poe_ch(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_poe_ch *poe_ch; + + if (!poe_is_supported(hr_dev) || hr_dev->caps.poe_ch_num <= 0) + goto out; + + poe_ch = kvcalloc(hr_dev->caps.poe_ch_num, + sizeof(struct hns_roce_poe_ch), GFP_KERNEL); + if (!poe_ch) + goto out; + + hr_dev->poe_ctx.poe_num = hr_dev->caps.poe_ch_num; + hr_dev->poe_ctx.poe_ch = poe_ch; + return; + +out: + hr_dev->poe_ctx.poe_num = 0; + hr_dev->poe_ctx.poe_ch = NULL; + +} + +static void hns_roce_unregister_poe_ch(struct hns_roce_dev *hr_dev) +{ + if (!poe_is_supported(hr_dev) || hr_dev->caps.poe_ch_num <= 0) + return; + + kvfree(hr_dev->poe_ctx.poe_ch); +} + static int hns_roce_alloc_dfx_cnt(struct hns_roce_dev *hr_dev) { hr_dev->dfx_cnt = kcalloc(HNS_ROCE_DFX_CNT_TOTAL, sizeof(atomic64_t), @@ -1485,6 +1541,7 @@ int hns_roce_init(struct hns_roce_dev *hr_dev) if (ret) goto error_failed_register_device; + hns_roce_register_poe_ch(hr_dev); hns_roce_register_debugfs(hr_dev); return 0; @@ -1522,6 +1579,7 @@ void hns_roce_exit(struct hns_roce_dev *hr_dev, bool bond_cleanup) hns_roce_unregister_sysfs(hr_dev); hns_roce_unregister_device(hr_dev, bond_cleanup); hns_roce_unregister_debugfs(hr_dev); + hns_roce_unregister_poe_ch(hr_dev); if (hr_dev->hw->hw_exit) hr_dev->hw->hw_exit(hr_dev); @@ -1536,6 +1594,8 @@ void hns_roce_exit(struct hns_roce_dev *hr_dev, bool bond_cleanup) if (hr_dev->hw->cmq_exit) hr_dev->hw->cmq_exit(hr_dev); hns_roce_dealloc_dfx_cnt(hr_dev); + if (hr_dev->notify_tbl) + kvfree(hr_dev->notify_tbl); } MODULE_LICENSE("Dual BSD/GPL"); diff --git a/drivers/infiniband/hw/hns/hns_roce_poe.c b/drivers/infiniband/hw/hns/hns_roce_poe.c new file mode 100644 index 0000000000000000000000000000000000000000..4de51eb9d138c0aa28521ae6f8b63aaab9e207e1 --- /dev/null +++ b/drivers/infiniband/hw/hns/hns_roce_poe.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* + * Copyright (c) 2023 Hisilicon Limited. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include "hns_roce_device.h" + +static int hns_roce_config_poe_ch(struct hns_roce_dev *hr_dev, u32 index, + u64 poe_addr) +{ + int ret; + + if (!hr_dev->hw->cfg_poe_ch) { + ibdev_err_ratelimited(&hr_dev->ib_dev, + "configure POE channel has not been supported in this device.\n"); + return -EOPNOTSUPP; + } + + ret = hr_dev->hw->cfg_poe_ch(hr_dev, index, poe_addr); + if (ret) + ibdev_err_ratelimited(&hr_dev->ib_dev, + "configure POE channel %u failed, ret = %d.\n", + index, ret); + + return ret; +} + +static bool check_poe_in_use(struct hns_roce_poe_ch *poe_ch) +{ + return poe_ch->en && refcount_read(&poe_ch->ref_cnt) > 1; +} + +static void update_poe_ch(struct hns_roce_poe_ch *poe_ch, u64 poe_addr) +{ + if (poe_addr) { + if (poe_addr != poe_ch->addr) + refcount_set(&poe_ch->ref_cnt, 1); + } else { + refcount_set(&poe_ch->ref_cnt, 0); + } + poe_ch->en = !!poe_addr; + poe_ch->addr = poe_addr; +} + +int hns_roce_register_poe_channel(struct hns_roce_dev *hr_dev, u8 channel, + u64 poe_addr) +{ + struct hns_roce_poe_ch *poe_ch; + int ret; + + if (!poe_is_supported(hr_dev)) + return -EOPNOTSUPP; + + if (channel >= hr_dev->poe_ctx.poe_num || !poe_addr) + return -EINVAL; + + poe_ch = &hr_dev->poe_ctx.poe_ch[channel]; + if (check_poe_in_use(poe_ch)) + return -EBUSY; + + ret = hns_roce_config_poe_ch(hr_dev, channel, poe_addr); + if (ret) + return ret; + + update_poe_ch(poe_ch, poe_addr); + + return ret; +} + +int hns_roce_unregister_poe_channel(struct hns_roce_dev *hr_dev, u8 channel) +{ + struct hns_roce_poe_ch *poe_ch; + int ret; + + if (!poe_is_supported(hr_dev)) + return -EOPNOTSUPP; + + if (channel >= hr_dev->poe_ctx.poe_num) + return -EINVAL; + + poe_ch = &hr_dev->poe_ctx.poe_ch[channel]; + if (check_poe_in_use(poe_ch)) + return -EBUSY; + + ret = hns_roce_config_poe_ch(hr_dev, channel, 0); + if (ret) + return ret; + + update_poe_ch(poe_ch, 0); + + return ret; +} diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index 77de664f9b7cd1810e7ae0b25ac831ef177cdceb..287523fceb70c8435609b76d6bce4b3079b5a038 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -1133,6 +1133,59 @@ static void set_congest_param(struct hns_roce_dev *hr_dev, default_congest_type(hr_dev, hr_qp); } +static void set_qp_notify_param(struct hns_roce_qp *hr_qp, + struct ib_cq *ib_cq) +{ + struct hns_roce_cq *hr_cq = ib_cq ? to_hr_cq(ib_cq) : NULL; + + /* + * Always enable write with notify for XRC TGT since no flag + * could be passed to kernel for this type of QP + */ + if ((hr_cq && hr_cq->flags & HNS_ROCE_CQ_FLAG_NOTIFY_EN) || + (hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT)) + hr_qp->en_flags |= HNS_ROCE_QP_CAP_WRITE_WITH_NOTIFY; +} + +static bool check_cq_poe_en(struct ib_cq *ib_cq) +{ + struct hns_roce_cq *hr_cq = ib_cq ? to_hr_cq(ib_cq) : NULL; + + return hr_cq && hr_cq->flags & HNS_ROCE_CQ_FLAG_POE_EN; +} + +static int set_uqp_create_flag_param(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp, + struct ib_qp_init_attr *init_attr, + struct hns_roce_ib_create_qp *ucmd) +{ + struct ib_device *ibdev = &hr_dev->ib_dev; + + if (check_cq_poe_en(init_attr->recv_cq) || + check_cq_poe_en(init_attr->send_cq)) { + if (!(ucmd->create_flags & + HNS_ROCE_CREATE_QP_FLAGS_STARS_MODE)) { + ibdev_err(ibdev, + "POE CQ only support STARS QP.\n"); + return -EINVAL; + } + } + + if (!(ucmd->comp_mask & HNS_ROCE_CREATE_QP_MASK_CREATE_FLAGS)) + return 0; + + if (ucmd->create_flags & HNS_ROCE_CREATE_QP_FLAGS_STARS_MODE) { + if (!check_cq_poe_en(init_attr->send_cq)) { + ibdev_err(ibdev, + "STARS QP SQ should be bound with POE CQ.\n"); + return -EINVAL; + } + + hr_qp->en_flags |= HNS_ROCE_QP_CAP_STARS_SQ_MODE; + } + return 0; +} + static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, struct ib_qp_init_attr *init_attr, struct ib_udata *udata, @@ -1160,6 +1213,8 @@ static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, if (init_attr->qp_type == IB_QPT_XRC_TGT) default_congest_type(hr_dev, hr_qp); + set_qp_notify_param(hr_qp, init_attr->recv_cq); + if (udata) { ret = ib_copy_from_udata(ucmd, udata, min(udata->inlen, sizeof(*ucmd))); @@ -1173,10 +1228,13 @@ static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, ibucontext); hr_qp->config = uctx->config; ret = set_user_sq_size(hr_dev, &init_attr->cap, hr_qp, ucmd); - if (ret) ibdev_err(ibdev, "Failed to set user SQ size, ret = %d\n", ret); + + ret = set_uqp_create_flag_param(hr_dev, hr_qp, init_attr, ucmd); + if (ret) + return ret; set_congest_param(hr_dev, hr_qp, ucmd); } else { if (init_attr->create_flags & @@ -1209,7 +1267,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, { struct hns_roce_ib_create_qp_resp resp = {}; struct ib_device *ibdev = &hr_dev->ib_dev; - struct hns_roce_ib_create_qp ucmd; + struct hns_roce_ib_create_qp ucmd = {}; int ret; mutex_init(&hr_qp->mutex); diff --git a/include/uapi/rdma/hns-abi.h b/include/uapi/rdma/hns-abi.h index a95b1d093f77f77dee550e868607e935f913225c..722313599d87af05a661cc7e2f8de5d92de273cc 100644 --- a/include/uapi/rdma/hns-abi.h +++ b/include/uapi/rdma/hns-abi.h @@ -36,15 +36,27 @@ #include +enum hns_roce_create_cq_create_flags { + HNS_ROCE_CREATE_CQ_FLAGS_POE_MODE = 1 << 0, + HNS_ROCE_CREATE_CQ_FLAGS_WRITE_WITH_NOTIFY = 1 << 1, +}; + struct hns_roce_ib_create_cq { __aligned_u64 buf_addr; __aligned_u64 db_addr; __u32 cqe_size; __u32 reserved; + __aligned_u64 create_flags; /* Use enum hns_roce_create_cq_create_flags */ + __u8 poe_channel; + __u8 notify_mode; + __u16 notify_idx; + __u16 rsv[2]; }; enum hns_roce_cq_cap_flags { HNS_ROCE_CQ_FLAG_RECORD_DB = 1 << 0, + HNS_ROCE_CQ_FLAG_POE_EN = 1 << 2, + HNS_ROCE_CQ_FLAG_NOTIFY_EN = 1 << 3, }; struct hns_roce_ib_create_cq_resp { @@ -78,6 +90,10 @@ enum hns_roce_create_qp_comp_mask { HNS_ROCE_CREATE_QP_MASK_CONGEST_TYPE = 1 << 1, }; +enum hns_roce_create_qp_flags { + HNS_ROCE_CREATE_QP_FLAGS_STARS_MODE = 1 << 0, +}; + enum hns_roce_congest_type_flags { HNS_ROCE_CREATE_QP_FLAGS_DCQCN = 1 << 0, HNS_ROCE_CREATE_QP_FLAGS_LDCP = 1 << 1, @@ -94,8 +110,8 @@ struct hns_roce_ib_create_qp { __u8 reserved[4]; __u8 pageshift; __aligned_u64 sdb_addr; - __aligned_u64 comp_mask; - __aligned_u64 create_flags; + __aligned_u64 comp_mask; /* Use enum hns_roce_create_qp_comp_mask */ + __aligned_u64 create_flags; /* Use enum hns_roce_create_qp_flags */ __aligned_u64 congest_type_flags; }; @@ -107,10 +123,12 @@ enum hns_roce_qp_cap_flags { HNS_ROCE_QP_CAP_DYNAMIC_CTX_ATTACH = 1 << 4, HNS_ROCE_QP_CAP_DIRECT_WQE = 1 << 5, HNS_ROCE_QP_CAP_DYNAMIC_CTX_DETACH = 1 << 6, + HNS_ROCE_QP_CAP_STARS_SQ_MODE = 1 << 7, + HNS_ROCE_QP_CAP_WRITE_WITH_NOTIFY = 1 << 8, }; struct hns_roce_ib_create_qp_resp { - __aligned_u64 cap_flags; + __aligned_u64 cap_flags; /* Use enum hns_roce_qp_cap_flags */ __aligned_u64 dwqe_mmap_key; }; @@ -232,4 +250,17 @@ enum hns_ib_dca_mem_query_attrs { HNS_IB_ATTR_DCA_MEM_QUERY_OUT_PAGE_COUNT, }; +#define HNS_IB_INVALID_ID 0XFFFF +struct hns_roce_ib_hw_id { + __u16 chip_id; + __u16 die_id; + __u16 func_id; + __u16 reserved; +}; + +struct hns_roce_ib_query_device_resp { + __u32 comp_mask; + __u32 len; + struct hns_roce_ib_hw_id hw_id; +}; #endif /* HNS_ABI_USER_H */