From c256e44d064225fe961dcf071666d20c01080f9d Mon Sep 17 00:00:00 2001 From: zhaoweibo Date: Thu, 19 Oct 2023 14:36:08 +0800 Subject: [PATCH 01/42] hns3 udma: support loading and unloading of udma driver. driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA ------------------------------------------------------- This patch adds the function of loading and unloading the udma driver. Signed-off-by: Xincheng Zhu Signed-off-by: Weibo Zhao --- arch/arm64/configs/openeuler_defconfig | 1 + drivers/ub/Kconfig | 1 + drivers/ub/Makefile | 1 + drivers/ub/hw/hns3/Kconfig | 17 + drivers/ub/hw/hns3/Makefile | 12 + drivers/ub/hw/hns3/hns3_udma_cmd.c | 302 +++++++++++++ drivers/ub/hw/hns3/hns3_udma_cmd.h | 45 ++ drivers/ub/hw/hns3/hns3_udma_common.h | 59 +++ drivers/ub/hw/hns3/hns3_udma_device.h | 297 +++++++++++++ drivers/ub/hw/hns3/hns3_udma_hem.h | 35 ++ drivers/ub/hw/hns3/hns3_udma_hw.c | 562 +++++++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_hw.h | 263 ++++++++++++ drivers/ub/hw/hns3/hns3_udma_main.c | 110 +++++ 13 files changed, 1705 insertions(+) create mode 100644 drivers/ub/hw/hns3/Kconfig create mode 100644 drivers/ub/hw/hns3/Makefile create mode 100644 drivers/ub/hw/hns3/hns3_udma_cmd.c create mode 100644 drivers/ub/hw/hns3/hns3_udma_cmd.h create mode 100644 drivers/ub/hw/hns3/hns3_udma_common.h create mode 100644 drivers/ub/hw/hns3/hns3_udma_device.h create mode 100644 drivers/ub/hw/hns3/hns3_udma_hem.h create mode 100644 drivers/ub/hw/hns3/hns3_udma_hw.c create mode 100644 drivers/ub/hw/hns3/hns3_udma_hw.h create mode 100644 drivers/ub/hw/hns3/hns3_udma_main.c diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig index 0c3e2a79be42..1e80e621ba28 100644 --- a/arch/arm64/configs/openeuler_defconfig +++ b/arch/arm64/configs/openeuler_defconfig @@ -6209,6 +6209,7 @@ CONFIG_ROH=m CONFIG_ROH_HNS=m CONFIG_UB=m CONFIG_UB_URMA=m +CONFIG_UB_UDMA_HNS3=m # end of Device Drivers # diff --git a/drivers/ub/Kconfig b/drivers/ub/Kconfig index 46651e1752b5..74640ce3c933 100644 --- a/drivers/ub/Kconfig +++ b/drivers/ub/Kconfig @@ -23,4 +23,5 @@ config UB_URMA need liburma from umdk . +source "drivers/ub/hw/hns3/Kconfig" endif # UB diff --git a/drivers/ub/Makefile b/drivers/ub/Makefile index 9cd43d23bffa..d60ee553690b 100644 --- a/drivers/ub/Makefile +++ b/drivers/ub/Makefile @@ -4,3 +4,4 @@ # obj-$(CONFIG_UB) += urma/ +obj-$(CONFIG_UB_UDMA_HNS3) += hw/hns3/ diff --git a/drivers/ub/hw/hns3/Kconfig b/drivers/ub/hw/hns3/Kconfig new file mode 100644 index 000000000000..82ca7b910673 --- /dev/null +++ b/drivers/ub/hw/hns3/Kconfig @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# UDMA_HNS configuration +# + +config UB_UDMA_HNS3 + default n + tristate "HNS3 UB UDMA Driver" + depends on UB && UB_URMA + depends on ARM64 + depends on UBL && PCI + help + This is a UB_UDMA_HNS3 driver for the Hisilicon UDMA engine. + This driver depands on ubcore and uburma. + This driver depands on UBL(UB link). The UDMA_HNS3 is a PCI device. + To compile UB_UDMA driver as module, choose M here. + module will be called hns3_udma. diff --git a/drivers/ub/hw/hns3/Makefile b/drivers/ub/hw/hns3/Makefile new file mode 100644 index 000000000000..bcd8cc619962 --- /dev/null +++ b/drivers/ub/hw/hns3/Makefile @@ -0,0 +1,12 @@ +# +## Makefile for the Huawei UDMA Linux driver. +# +## + +MODULE_NAME := hns3_udma +ccflags-y += -I$(srctree)/drivers/net/ethernet/hisilicon/hns3/ \ + -I$(srctree)/drivers/ub/urma/ + +$(MODULE_NAME)-objs := hns3_udma_hw.o hns3_udma_main.o hns3_udma_cmd.o + +obj-$(CONFIG_UB_UDMA_HNS3) := hns3_udma.o diff --git a/drivers/ub/hw/hns3/hns3_udma_cmd.c b/drivers/ub/hw/hns3/hns3_udma_cmd.c new file mode 100644 index 000000000000..27a936e0b800 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_cmd.c @@ -0,0 +1,302 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include +#include +#include +#include "hnae3.h" +#include "hns3_udma_common.h" +#include "hns3_udma_cmd.h" + +int udma_cmd_init(struct udma_dev *udma_dev) +{ + sema_init(&udma_dev->cmd.poll_sem, 1); + udma_dev->cmd.use_events = 0; + udma_dev->cmd.max_cmds = CMD_MAX_NUM; + udma_dev->cmd.pool = dma_pool_create("udma_cmd", udma_dev->dev, + UDMA_MAILBOX_SIZE, + UDMA_MAILBOX_SIZE, 0); + if (!udma_dev->cmd.pool) + return -ENOMEM; + + init_rwsem(&udma_dev->cmd.udma_mb_rwsem); + + return 0; +} + +void udma_cmd_cleanup(struct udma_dev *udma_dev) +{ + down_write(&udma_dev->cmd.udma_mb_rwsem); + dma_pool_destroy(udma_dev->cmd.pool); + up_write(&udma_dev->cmd.udma_mb_rwsem); +} + +static uint32_t udma_cmd_hw_reseted(struct udma_dev *dev, + uint64_t instance_stage, + uint64_t reset_stage) +{ + /* When hardware reset has been completed once or more, we should stop + * sending mailbox&cmq&doorbell to hardware. If now in .init_instance() + * function, we should exit with error. If now at HNAE3_INIT_CLIENT + * stage of soft reset process, we should exit with error, and then + * HNAE3_INIT_CLIENT related process can rollback the operation like + * notifing hardware to free resources, HNAE3_INIT_CLIENT related + * process will exit with error to notify NIC driver to reschedule soft + * reset process once again. + */ + dev->is_reset = true; + dev->dis_db = true; + + if (reset_stage == UDMA_STATE_RST_INIT || + instance_stage == UDMA_STATE_INIT) + return CMD_RST_PRC_EBUSY; + + return CMD_RST_PRC_SUCCESS; +} + +static uint32_t udma_cmd_hw_resetting(struct udma_dev *dev, + uint64_t instance_stage, + uint64_t reset_stage) +{ +#define HW_RESET_TIMEOUT_US 1000000 +#define HW_RESET_DELAY_US 1 + struct udma_priv *priv = (struct udma_priv *)dev->priv; + struct hnae3_handle *handle = priv->handle; + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + uint64_t val; + int ret; + + /* When hardware reset is detected, we should stop sending mailbox&cmq& + * doorbell to hardware. If now in .init_instance() function, we should + * exit with error. If now at HNAE3_INIT_CLIENT stage of soft reset + * process, we should exit with error, and then HNAE3_INIT_CLIENT + * related process can rollback the operation like notifing hardware to + * free resources, HNAE3_INIT_CLIENT related process will exit with + * error to notify UBNIC driver to reschedule soft reset process once + * again. + */ + dev->dis_db = true; + + ret = read_poll_timeout_atomic(ops->ae_dev_reset_cnt, val, + val > dev->reset_cnt, HW_RESET_DELAY_US, + HW_RESET_TIMEOUT_US, false, handle); + if (!ret) + dev->is_reset = true; + + if (!dev->is_reset || reset_stage == UDMA_STATE_RST_INIT || + instance_stage == UDMA_STATE_INIT) + return CMD_RST_PRC_EBUSY; + + return CMD_RST_PRC_SUCCESS; +} + +static uint32_t udma_cmd_sw_resetting(struct udma_dev *dev) +{ + struct udma_priv *priv = (struct udma_priv *)dev->priv; + struct hnae3_handle *handle = priv->handle; + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + + /* When software reset is detected at .init_instance() function, we + * should stop sending mailbox&cmq&doorbell to hardware, and exit + * with error. + */ + dev->dis_db = true; + if (ops->ae_dev_reset_cnt(handle) != dev->reset_cnt) + dev->is_reset = true; + + return CMD_RST_PRC_EBUSY; +} + +static uint32_t check_aedev_reset_status(struct udma_dev *dev, + struct hnae3_handle *handle) +{ + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + uint64_t instance_stage; + uint64_t reset_stage; + uint64_t reset_cnt; + bool sw_resetting; + bool hw_resetting; + + /* The meaning of the following variables: + * reset_cnt -- The count value of completed hardware reset. + * hw_resetting -- Whether hardware device is resetting now. + * sw_resetting -- Whether UBNIC's software reset process is + * running now. + */ + instance_stage = handle->udmainfo.instance_state; + reset_stage = handle->udmainfo.reset_state; + reset_cnt = ops->ae_dev_reset_cnt(handle); + if (reset_cnt != dev->reset_cnt) + return udma_cmd_hw_reseted(dev, instance_stage, reset_stage); + + hw_resetting = ops->get_cmdq_stat(handle); + if (hw_resetting) + return udma_cmd_hw_resetting(dev, instance_stage, reset_stage); + + sw_resetting = ops->ae_dev_resetting(handle); + if (sw_resetting && instance_stage == UDMA_STATE_INIT) + return udma_cmd_sw_resetting(dev); + + return CMD_RST_PRC_OTHERS; +} + +bool udma_chk_mbox_is_avail(struct udma_dev *dev, bool *busy) +{ + struct udma_priv *priv = (struct udma_priv *)dev->priv; + uint32_t status; + + /* + * CMDQ fatal err or system reset means hw/fw is in an abnormal or + * unresponsive state, CMDQ command needs to return quickly, + * in order to avoid excessive printing, return CMD_RST_PRC_SUCCESS. + */ + if (dev->cmd.state == UDMA_CMDQ_STATE_FATAL_ERR || + dev->is_reset) + status = CMD_RST_PRC_SUCCESS; + else + status = check_aedev_reset_status(dev, priv->handle); + + *busy = (status == CMD_RST_PRC_EBUSY); + + return status == CMD_RST_PRC_OTHERS; +} + +static int udma_cmq_csq_done(struct udma_dev *dev) +{ + struct udma_priv *priv = (struct udma_priv *)dev->priv; + uint32_t tail = ub_read(dev, UDMA_TX_CMQ_CI_REG); + + return tail == priv->cmq.csq.head; +} + +void udma_cmq_setup_basic_desc(struct udma_cmq_desc *desc, + enum udma_opcode_type opcode, + bool is_read) +{ + memset((void *)desc, 0, sizeof(struct udma_cmq_desc)); + desc->opcode = cpu_to_le16(opcode); + desc->flag = + cpu_to_le16(UDMA_CMD_FLAG_NO_INTR | UDMA_CMD_FLAG_IN); + if (is_read) + desc->flag |= cpu_to_le16(UDMA_CMD_FLAG_WR); + else + desc->flag &= cpu_to_le16(~UDMA_CMD_FLAG_WR); +} + +void dump_desc(struct udma_dev *dev, + struct udma_cmq_desc *desc) +{ + if (desc->opcode == UDMA_OPC_QUERY_MB_ST) + return; + + if (((desc->data[SUB_OPCODE_IDX] & 0xFF) == + UDMA_CMD_WRITE_QPC_TIMER_BT0) || + ((desc->data[SUB_OPCODE_IDX] & 0xFF) == + UDMA_CMD_WRITE_CQC_TIMER_BT0)) + dev_err_ratelimited(dev->dev, + "Send cmd opcode:0x%4x, data: %08x %08x %08x %08x %08x %08x\n", + desc->opcode, desc->data[0], + desc->data[1], desc->data[2], + desc->data[3], desc->data[4], desc->data[5]); + else + dev_info(dev->dev, + "Send cmd opcode:0x%4x, data: %08x %08x %08x %08x %08x %08x\n", + desc->opcode, desc->data[0], + desc->data[1], desc->data[2], + desc->data[3], desc->data[4], desc->data[5]); +} + +static int __udma_cmq_send(struct udma_dev *dev, struct udma_cmq_desc *desc, + int num) +{ + struct udma_priv *priv = (struct udma_priv *)dev->priv; + struct udma_cmq_ring *csq = &priv->cmq.csq; + uint32_t timeout = 0; + uint16_t desc_ret; + uint32_t tail; + int ret = 0; + int i; + + tail = csq->head; + + mutex_lock(&csq->lock); + + for (i = 0; i < num; i++) { + csq->desc[csq->head++] = desc[i]; + if (csq->head == csq->desc_num) + csq->head = 0; + } + + /* barrier */ + wmb(); + + /* Write to hardware */ + ub_write(dev, UDMA_TX_CMQ_PI_REG, csq->head); + + do { + if (udma_cmq_csq_done(dev)) + break; + udelay(1); + } while (++timeout < priv->cmq.tx_timeout); + + if (udma_cmq_csq_done(dev)) { + for (i = 0; i < num; i++) { + /* check the result of hardware write back */ + desc[i] = csq->desc[tail++]; + if (tail == csq->desc_num) + tail = 0; + + desc_ret = le16_to_cpu(desc[i].retval); + if (likely(desc_ret == CMD_EXEC_SUCCESS)) + continue; + + dev_err_ratelimited(dev->dev, + "CMDQ IO error, opcode = %x, return = %x\n", + desc->opcode, desc_ret); + ret = -EIO; + } + } else { + /* FW/HW reset or incorrect number of desc */ + tail = ub_read(dev, UDMA_TX_CMQ_CI_REG); + dev_warn(dev->dev, "CMDQ move tail from %d to %d\n", + csq->head, tail); + csq->head = tail; + + dev->cmd.state = UDMA_CMDQ_STATE_HEAD_TAIL_ERR; + + ret = -EAGAIN; + } + + mutex_unlock(&csq->lock); + + return ret; +} + +int udma_cmq_send(struct udma_dev *dev, struct udma_cmq_desc *desc, int num) +{ + bool busy; + int ret; + + if (!udma_chk_mbox_is_avail(dev, &busy)) + return busy ? -EBUSY : 0; + + ret = __udma_cmq_send(dev, desc, num); + if (ret) { + if (!udma_chk_mbox_is_avail(dev, &busy)) + return busy ? -EBUSY : 0; + } + + return ret; +} diff --git a/drivers/ub/hw/hns3/hns3_udma_cmd.h b/drivers/ub/hw/hns3/hns3_udma_cmd.h new file mode 100644 index 000000000000..793a989494cb --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_cmd.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_CMD_H +#define _UDMA_CMD_H + +#include "hns3_udma_device.h" + +#define CMD_MAX_NUM 32 +#define UDMA_MAILBOX_SIZE 4096 +#define UDMA_CMD_TIMEOUT_MSECS 10000 +#define CMD_POLL_TOKEN 0xffff +#define SUB_OPCODE_IDX 4 + +enum { + /* CQC TIMER commands */ + UDMA_CMD_WRITE_CQC_TIMER_BT0 = 0x23, + + /* QPC TIMER commands */ + UDMA_CMD_WRITE_QPC_TIMER_BT0 = 0x33, +}; + +enum { + CMD_RST_PRC_OTHERS, + CMD_RST_PRC_SUCCESS, + CMD_RST_PRC_EBUSY, +}; + +void udma_cmq_setup_basic_desc(struct udma_cmq_desc *desc, + enum udma_opcode_type opcode, + bool is_read); +int udma_cmq_send(struct udma_dev *dev, struct udma_cmq_desc *desc, int num); +#endif /* _UDMA_CMD_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_common.h b/drivers/ub/hw/hns3/hns3_udma_common.h new file mode 100644 index 000000000000..e726c550ba1f --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_common.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_COMMON_H +#define _UDMA_COMMON_H +#include + +#define ub_write(dev, reg, val) writel((val), (dev)->reg_base + (reg)) +#define ub_read(dev, reg) readl((dev)->reg_base + (reg)) + +#define UDMA_UDP_DPORT 4791 + +#define udma_get_field(origin, mask, shift) \ + ((le32_to_cpu(origin) & (mask)) >> (uint32_t)(shift)) + +#define udma_set_field(origin, mask, shift, val) \ + do { \ + (origin) &= ~cpu_to_le32(mask); \ + (origin) |= cpu_to_le32(((uint32_t)(val) << \ + (uint32_t)(shift)) & (mask)); \ + } while (0) + +#define udma_set_bit(origin, shift, val) \ + udma_set_field((origin), (1ul << (shift)), (shift), (val)) + +#define _udma_reg_clear(ptr, field) \ + ({ \ + const uint32_t *_ptr = (uint32_t *)(ptr); \ + BUILD_BUG_ON((((field) >> 32) / 32) != ((((field) << 32) >> 32) / 32)); \ + *((uint32_t *)_ptr + ((field) >> 32) / 32) &= \ + ~cpu_to_le32(GENMASK(((field) >> 32) % 32, (((field) << 32) >> 32) % 32)); \ + }) + +#define udma_reg_clear(ptr, field) _udma_reg_clear(ptr, field) + +#define _udma_reg_write(ptr, field, val) \ + ({ \ + uint32_t _val = val; \ + _udma_reg_clear((ptr), field); \ + *((uint32_t *)(ptr) + ((field) >> 32) / 32) |= \ + cpu_to_le32(FIELD_PREP(GENMASK(((field) >> 32) % 32, \ + (((field) << 32) >> 32) % 32), _val)); \ + }) + +#define udma_reg_write(ptr, field, val) _udma_reg_write(ptr, field, val) + +#endif /* _UDMA_COMMON_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_device.h b/drivers/ub/hw/hns3/hns3_udma_device.h new file mode 100644 index 000000000000..9df2aba5d94a --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_device.h @@ -0,0 +1,297 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_DEVICE_H +#define _UDMA_DEVICE_H + +#include +#include +#include "hns3_udma_hw.h" +#include "hns3_udma_common.h" + +#define UDMA_MAX_PORTS 6 +#define UDMA_INVALID_ID 0xffff +#define UDMA_MAX_IRQ_NUM 128 +#define UDMA_CAP_FLAGS_EX_SHIFT 12 +#define UDMA_TX_CMQ_PI_REG 0x07010 +#define UDMA_TX_CMQ_CI_REG 0x07014 + +enum udma_reset_stage { + UDMA_STATE_RST_DOWN = 2, + UDMA_STATE_RST_UNINIT, + UDMA_STATE_RST_INIT, + UDMA_STATE_RST_INITED, +}; + +enum udma_instance_state { + UDMA_STATE_NON_INIT, + UDMA_STATE_INIT, + UDMA_STATE_INITED, + UDMA_STATE_UNINIT, +}; + +struct udma_cmd_context { + struct completion done; + int next; + uint16_t token; + uint16_t busy; +}; + +struct udma_cmq_desc { + uint16_t opcode; + uint16_t flag; + uint16_t retval; + uint16_t rsv; + uint32_t data[6]; +}; + +enum udma_cmdq_state { + UDMA_CMDQ_STATE_HEAD_TAIL_ERR = 1, + UDMA_CMDQ_STATE_FATAL_ERR, +}; + +struct udma_cmq_ring { + dma_addr_t desc_dma_addr; + struct udma_cmq_desc *desc; + uint32_t head; + uint16_t desc_num; + uint8_t flag; + struct mutex lock; +}; + +struct udma_cmdq { + struct dma_pool *pool; + struct semaphore poll_sem; + struct semaphore event_sem; + int max_cmds; + spinlock_t ctx_lock; + int free_head; + struct udma_cmd_context *context; + /* + * Process whether use event mode, init default non-zero + * After the event queue of cmd event ready, + * can switch into event mode + * close device, switch into poll mode(non event mode) + */ + uint8_t use_events; + struct rw_semaphore udma_mb_rwsem; + enum udma_cmdq_state state; +}; +struct udma_netdev { + spinlock_t lock; + struct net_device *netdevs[UDMA_MAX_PORTS]; +}; + +enum udma_device_state { + UDMA_DEVICE_STATE_RST_DOWN = 1, + UDMA_DEVICE_STATE_UNINIT, +}; + +struct udma_cmq { + struct udma_cmq_ring csq; + uint16_t tx_timeout; +}; + +struct udma_priv { + struct hnae3_handle *handle; + struct udma_cmq cmq; +}; + +struct udma_dev; +struct udma_hw { + int (*cmq_init)(struct udma_dev *udma_dev); + void (*cmq_exit)(struct udma_dev *udma_dev); + int (*hw_profile)(struct udma_dev *udma_dev); + int (*hw_init)(struct udma_dev *udma_dev); + void (*hw_exit)(struct udma_dev *udma_dev); +}; + +struct udma_caps { + uint64_t fw_ver; + uint8_t num_ports; + int gid_table_len[UDMA_MAX_PORTS]; + int pkey_table_len[UDMA_MAX_PORTS]; + int local_ca_ack_delay; + int num_uars; + uint32_t phy_num_uars; + uint32_t max_sq_sg; + uint32_t max_sq_inline; + uint32_t max_rq_sg; + uint32_t max_extend_sg; + uint32_t num_qps; + uint32_t num_qps_shift; + uint32_t num_pi_qps; + uint32_t reserved_qps; + int num_qpc_timer; + int num_srqs; + uint32_t max_wqes; + uint32_t max_srq_wrs; + uint32_t max_srq_sges; + uint32_t max_sq_desc_sz; + uint32_t max_rq_desc_sz; + uint32_t max_srq_desc_sz; + int max_qp_init_rdma; + int max_qp_dest_rdma; + uint32_t num_cqs; + uint32_t max_cqes; + uint32_t min_cqes; + uint32_t min_wqes; + uint32_t reserved_cqs; + int reserved_srqs; + int num_aeq_vectors; + int num_comp_vectors; + int num_other_vectors; + uint32_t num_mtpts; + uint32_t num_mtt_segs; + uint32_t num_srqwqe_segs; + uint32_t num_idx_segs; + int reserved_mrws; + int reserved_uars; + int num_pds; + int reserved_pds; + uint32_t num_xrcds; + uint32_t reserved_xrcds; + uint32_t mtt_entry_sz; + uint32_t cqe_sz; + uint32_t page_size_cap; + uint32_t reserved_lkey; + int mtpt_entry_sz; + int qpc_sz; + int irrl_entry_sz; + int trrl_entry_sz; + int cqc_entry_sz; + int scc_ctx_sz; + int qpc_timer_entry_sz; + int cqc_timer_entry_sz; + int srqc_entry_sz; + int idx_entry_sz; + uint32_t pbl_ba_pg_sz; + uint32_t pbl_buf_pg_sz; + uint32_t pbl_hop_num; + int aeqe_depth; + int ceqe_depth; + uint32_t aeqe_size; + uint32_t ceqe_size; + enum ubcore_mtu max_mtu; + uint32_t qpc_bt_num; + uint32_t qpc_timer_bt_num; + uint32_t srqc_bt_num; + uint32_t cqc_bt_num; + uint32_t cqc_timer_bt_num; + uint32_t mpt_bt_num; + uint32_t eqc_bt_num; + uint32_t smac_bt_num; + uint32_t sgid_bt_num; + uint32_t sccc_bt_num; + uint32_t gmv_bt_num; + uint32_t qpc_ba_pg_sz; + uint32_t qpc_buf_pg_sz; + uint32_t qpc_hop_num; + uint32_t srqc_ba_pg_sz; + uint32_t srqc_buf_pg_sz; + uint32_t srqc_hop_num; + uint32_t cqc_ba_pg_sz; + uint32_t cqc_buf_pg_sz; + uint32_t cqc_hop_num; + uint32_t mpt_ba_pg_sz; + uint32_t mpt_buf_pg_sz; + uint32_t mpt_hop_num; + uint32_t mtt_ba_pg_sz; + uint32_t mtt_buf_pg_sz; + uint32_t mtt_hop_num; + uint32_t wqe_sq_hop_num; + uint32_t wqe_sge_hop_num; + uint32_t wqe_rq_hop_num; + uint32_t sccc_ba_pg_sz; + uint32_t sccc_buf_pg_sz; + uint32_t sccc_hop_num; + uint32_t qpc_timer_ba_pg_sz; + uint32_t qpc_timer_buf_pg_sz; + uint32_t qpc_timer_hop_num; + uint32_t cqc_timer_ba_pg_sz; + uint32_t cqc_timer_buf_pg_sz; + uint32_t cqc_timer_hop_num; + uint32_t cqe_ba_pg_sz; + uint32_t cqe_buf_pg_sz; + uint32_t cqe_hop_num; + uint32_t srqwqe_ba_pg_sz; + uint32_t srqwqe_buf_pg_sz; + uint32_t srqwqe_hop_num; + uint32_t idx_ba_pg_sz; + uint32_t idx_buf_pg_sz; + uint32_t idx_hop_num; + uint32_t eqe_ba_pg_sz; + uint32_t eqe_buf_pg_sz; + uint32_t eqe_hop_num; + uint32_t gmv_entry_num; + uint32_t gmv_entry_sz; + uint32_t gmv_ba_pg_sz; + uint32_t gmv_buf_pg_sz; + uint32_t gmv_hop_num; + uint32_t sl_num; + uint32_t llm_buf_pg_sz; + uint32_t llm_ba_idx; + uint32_t llm_ba_num; + uint32_t chunk_sz; /* chunk size in non multihop mode */ + uint64_t flags; + uint16_t default_ceq_max_cnt; + uint16_t default_ceq_period; + uint16_t default_aeq_max_cnt; + uint16_t default_aeq_period; + uint16_t default_aeq_arm_st; + uint16_t default_ceq_arm_st; + uint8_t cong_type; + uint8_t oor_en; + uint8_t reorder_cq_buffer_en; + uint8_t reorder_cap; + uint8_t reorder_cq_shift; + uint32_t onflight_size; + uint8_t dynamic_ack_timeout; + uint32_t num_jfc_shift; + uint32_t num_jfs_shift; + uint32_t num_jfr_shift; + uint32_t num_jetty_shift; + uint8_t poe_ch_num; +}; + +struct udma_dev { + struct ubcore_device ub_dev; + struct pci_dev *pci_dev; + struct device *dev; + + bool is_reset; + bool dis_db; + uint64_t reset_cnt; + struct udma_netdev uboe; + uint8_t __iomem *reg_base; + struct udma_caps caps; + + int irq_num; + int irq[UDMA_MAX_IRQ_NUM]; + char dev_name[UBCORE_MAX_DEV_NAME]; + uint64_t sys_image_guid; + struct udma_cmdq cmd; + int cmd_mod; + const struct udma_hw *hw; + void *priv; + uint16_t func_id; + uint32_t func_num; + uint32_t cong_algo_tmpl_id; +}; + +int udma_hnae_client_init(struct udma_dev *udma_dev); +void udma_hnae_client_exit(struct udma_dev *udma_dev); + +#endif /* _UDMA_DEVICE_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_hem.h b/drivers/ub/hw/hns3/hns3_udma_hem.h new file mode 100644 index 000000000000..1bef0304fbec --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_hem.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_HEM_H +#define _UDMA_HEM_H + +enum { + /* UDMA MAP HEM(Hardware Entry Memory) */ + HEM_TYPE_QPC = 0, + HEM_TYPE_MTPT, + HEM_TYPE_CQC, + HEM_TYPE_SRQC, + /* SCC CONTEXT */ + HEM_TYPE_SCCC, + + /* UDMA UNMAP HEM */ + HEM_TYPE_MTT, + HEM_TYPE_CQE, + HEM_TYPE_SRQWQE, + HEM_TYPE_INDEX, +}; + +#endif diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.c b/drivers/ub/hw/hns3/hns3_udma_hw.c new file mode 100644 index 000000000000..7fcbe31c07b8 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_hw.c @@ -0,0 +1,562 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include +#include +#include +#include "hnae3.h" +#include "hns3_udma_cmd.h" + +static const struct pci_device_id udma_hw_pci_tbl[] = { + { PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_UDMA_OVER_UBL), + HNAE3_DEV_SUPPORT_UDMA_OVER_UBL_DCB_BITS }, + { PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_UDMA), + HNAE3_DEV_SUPPORT_UDMA_DCB_BITS }, + /* required last entry */ + {} +}; + +static int udma_cmq_query_hw_info(struct udma_dev *udma_dev) +{ + struct udma_query_version *resp; + struct udma_cmq_desc desc; + int ret; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_QUERY_HW_VER, true); + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) + return ret; + + resp = (struct udma_query_version *)desc.data; + + return ret; +} + + +static int udma_query_fw_ver(struct udma_dev *udma_dev) +{ + struct udma_query_fw_info *resp; + struct udma_cmq_desc desc; + int ret; + + udma_cmq_setup_basic_desc(&desc, UDMA_QUERY_FW_VER, true); + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) + return ret; + + resp = (struct udma_query_fw_info *)desc.data; + udma_dev->caps.fw_ver = (uint64_t)(le32_to_cpu(resp->fw_ver)); + + return 0; +} + +static int udma_query_func_id(struct udma_dev *udma_dev) +{ + struct udma_hw_id_query_cmq *resp; + struct udma_cmq_desc desc; + int ret; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_QUERY_HW_ID, true); + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) { + if (desc.retval != CMD_NOT_EXIST) + dev_warn(udma_dev->dev, + "failed to query hw id, ret = %d.\n", ret); + + goto invalid_val; + } + + resp = (struct udma_hw_id_query_cmq *)desc.data; + udma_dev->func_id = (uint16_t)le32_to_cpu(resp->func_id); + return 0; + +invalid_val: + udma_dev->func_id = UDMA_INVALID_ID; + return ret; +} + +static int udma_query_func_info(struct udma_dev *udma_dev) +{ + struct udma_pf_func_info *resp; + struct udma_cmq_desc desc; + int ret; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_QUERY_FUNC_INFO, true); + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) + return ret; + + resp = (struct udma_pf_func_info *)desc.data; + udma_dev->func_num = le32_to_cpu(resp->own_func_num); + udma_dev->cong_algo_tmpl_id = le32_to_cpu(resp->own_mac_id); + + return udma_query_func_id(udma_dev); +} + +static int udma_config_global_param(struct udma_dev *udma_dev) +{ + struct udma_cmq_desc desc; + struct udma_cmq_req *req = (struct udma_cmq_req *)desc.data; + uint32_t clock_cycles_of_1us; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_CFG_GLOBAL_PARAM, + false); + + clock_cycles_of_1us = UDMA_1US_CFG; + + udma_reg_write(req, CFG_GLOBAL_PARAM_1US_CYCLES, clock_cycles_of_1us); + udma_reg_write(req, CFG_GLOBAL_PARAM_UDP_PORT, UDMA_UDP_DPORT); + + return udma_cmq_send(udma_dev, &desc, 1); +} + +static int __udma_set_vf_switch_param(struct udma_dev *udma_dev, + uint32_t vf_id) +{ + struct udma_vf_switch *swt; + struct udma_cmq_desc desc; + int ret; + + swt = (struct udma_vf_switch *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_SWITCH_PARAMETER_CFG, true); + swt->udma_sel |= cpu_to_le32(NIC_ICL_SWITCH_CMD_UDMA_SEL); + udma_set_field(swt->fun_id, VF_SWITCH_DATA_FUN_ID_VF_ID_M, + VF_SWITCH_DATA_FUN_ID_VF_ID_S, vf_id); + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) + return ret; + + desc.flag = + cpu_to_le16(UDMA_CMD_FLAG_NO_INTR | UDMA_CMD_FLAG_IN); + desc.flag &= cpu_to_le16(~UDMA_CMD_FLAG_WR); + udma_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LPBK_S, 1); + udma_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LCL_LPBK_S, 0); + udma_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_DST_OVRD_S, 1); + + return udma_cmq_send(udma_dev, &desc, 1); +} + +static int udma_set_vf_switch_param(struct udma_dev *udma_dev) +{ + uint32_t vf_id; + int ret; + + for (vf_id = 0; vf_id < udma_dev->func_num; vf_id++) { + ret = __udma_set_vf_switch_param(udma_dev, vf_id); + if (ret) + return ret; + } + return 0; +} + +static int udma_query_caps(struct udma_dev *udma_dev) +{ + enum udma_opcode_type opcode = UDMA_OPC_QUERY_PF_CAPS_NUM; + struct udma_cmq_desc desc[UDMA_QUERY_PF_CAPS_CMD_NUM]; + struct udma_caps *caps = &udma_dev->caps; + struct udma_query_pf_caps_a *resp_a; + struct udma_query_pf_caps_b *resp_b; + struct udma_query_pf_caps_c *resp_c; + struct udma_query_pf_caps_d *resp_d; + struct udma_query_pf_caps_e *resp_e; + int ctx_hop_num; + int pbl_hop_num; + int ret; + int i; + + for (i = 0; i < UDMA_QUERY_PF_CAPS_CMD_NUM; i++) { + udma_cmq_setup_basic_desc(&desc[i], opcode, true); + if (i < (UDMA_QUERY_PF_CAPS_CMD_NUM - 1)) + desc[i].flag |= cpu_to_le16(UDMA_CMD_FLAG_NEXT); + else + desc[i].flag &= ~cpu_to_le16(UDMA_CMD_FLAG_NEXT); + } + + ret = udma_cmq_send(udma_dev, desc, UDMA_QUERY_PF_CAPS_CMD_NUM); + if (ret) + return ret; + + resp_a = (struct udma_query_pf_caps_a *)desc[0].data; + resp_b = (struct udma_query_pf_caps_b *)desc[1].data; + resp_c = (struct udma_query_pf_caps_c *)desc[2].data; + resp_d = (struct udma_query_pf_caps_d *)desc[3].data; + resp_e = (struct udma_query_pf_caps_e *)desc[4].data; + + caps->local_ca_ack_delay = resp_a->local_ca_ack_delay; + caps->max_sq_sg = le16_to_cpu(resp_a->max_sq_sg); + caps->max_sq_inline = le16_to_cpu(resp_a->max_sq_inline); + caps->max_rq_sg = le16_to_cpu(resp_a->max_rq_sg); + caps->max_rq_sg = roundup_pow_of_two(caps->max_rq_sg); + caps->max_extend_sg = le32_to_cpu(resp_a->max_extend_sg); + caps->num_qpc_timer = le16_to_cpu(resp_a->num_qpc_timer); + caps->max_srq_sges = le16_to_cpu(resp_a->max_srq_sges); + /* reserved for UM header */ + caps->max_srq_sges = roundup_pow_of_two(caps->max_srq_sges) - 1; + caps->num_aeq_vectors = resp_a->num_aeq_vectors; + caps->num_other_vectors = resp_a->num_other_vectors; + caps->max_sq_desc_sz = resp_a->max_sq_desc_sz; + caps->max_rq_desc_sz = resp_a->max_rq_desc_sz; + caps->max_srq_desc_sz = resp_a->max_srq_desc_sz; + caps->cqe_sz = resp_a->cqe_sz; + + caps->mtpt_entry_sz = resp_b->mtpt_entry_sz; + caps->irrl_entry_sz = resp_b->irrl_entry_sz; + caps->trrl_entry_sz = resp_b->trrl_entry_sz; + caps->cqc_entry_sz = resp_b->cqc_entry_sz; + caps->srqc_entry_sz = resp_b->srqc_entry_sz; + caps->idx_entry_sz = resp_b->idx_entry_sz; + caps->scc_ctx_sz = resp_b->sccc_sz; + caps->max_mtu = (enum ubcore_mtu)resp_b->max_mtu; + caps->qpc_sz = le16_to_cpu(resp_b->qpc_sz); + caps->min_cqes = resp_b->min_cqes; + caps->min_wqes = resp_b->min_wqes; + caps->page_size_cap = le32_to_cpu(resp_b->page_size_cap); + caps->pkey_table_len[0] = resp_b->pkey_table_len; + caps->phy_num_uars = resp_b->phy_num_uars; + ctx_hop_num = resp_b->ctx_hop_num; + pbl_hop_num = resp_b->pbl_hop_num; + + caps->num_pds = 1 << udma_get_field(resp_c->cap_flags_num_pds, + QUERY_PF_CAPS_C_NUM_PDS_M, + QUERY_PF_CAPS_C_NUM_PDS_S); + caps->flags = udma_get_field(resp_c->cap_flags_num_pds, + QUERY_PF_CAPS_C_CAP_FLAGS_M, + QUERY_PF_CAPS_C_CAP_FLAGS_S); + caps->flags |= le16_to_cpu(resp_d->cap_flags_ex) << + UDMA_CAP_FLAGS_EX_SHIFT; + + caps->num_cqs = 1 << udma_get_field(resp_c->max_gid_num_cqs, + QUERY_PF_CAPS_C_NUM_CQS_M, + QUERY_PF_CAPS_C_NUM_CQS_S); + caps->gid_table_len[0] = udma_get_field(resp_c->max_gid_num_cqs, + QUERY_PF_CAPS_C_MAX_GID_M, + QUERY_PF_CAPS_C_MAX_GID_S); + + caps->max_cqes = 1 << udma_get_field(resp_c->cq_depth, + QUERY_PF_CAPS_C_CQ_DEPTH_M, + QUERY_PF_CAPS_C_CQ_DEPTH_S); + caps->num_mtpts = 1 << udma_get_field(resp_c->num_mrws, + QUERY_PF_CAPS_C_NUM_MRWS_M, + QUERY_PF_CAPS_C_NUM_MRWS_S); + caps->num_qps = 1 << udma_get_field(resp_c->ord_num_qps, + QUERY_PF_CAPS_C_NUM_QPS_M, + QUERY_PF_CAPS_C_NUM_QPS_S); + caps->num_qps_shift = udma_get_field(resp_c->ord_num_qps, + QUERY_PF_CAPS_C_NUM_QPS_M, + QUERY_PF_CAPS_C_NUM_QPS_S); + caps->max_qp_init_rdma = udma_get_field(resp_c->ord_num_qps, + QUERY_PF_CAPS_C_MAX_ORD_M, + QUERY_PF_CAPS_C_MAX_ORD_S); + caps->max_qp_dest_rdma = caps->max_qp_init_rdma; + caps->max_wqes = 1 << le16_to_cpu(resp_c->sq_depth); + caps->num_srqs = 1 << udma_get_field(resp_d->wq_hop_num_max_srqs, + QUERY_PF_CAPS_D_NUM_SRQS_M, + QUERY_PF_CAPS_D_NUM_SRQS_S); + caps->cong_type = udma_get_field(resp_d->wq_hop_num_max_srqs, + QUERY_PF_CAPS_D_CONG_TYPE_M, + QUERY_PF_CAPS_D_CONG_TYPE_S); + caps->max_srq_wrs = 1 << le16_to_cpu(resp_d->srq_depth); + + caps->ceqe_depth = 1 << udma_get_field(resp_d->num_ceqs_ceq_depth, + QUERY_PF_CAPS_D_CEQ_DEPTH_M, + QUERY_PF_CAPS_D_CEQ_DEPTH_S); + caps->num_comp_vectors = udma_get_field(resp_d->num_ceqs_ceq_depth, + QUERY_PF_CAPS_D_NUM_CEQS_M, + QUERY_PF_CAPS_D_NUM_CEQS_S); + + caps->aeqe_depth = 1 << udma_get_field(resp_d->arm_st_aeq_depth, + QUERY_PF_CAPS_D_AEQ_DEPTH_M, + QUERY_PF_CAPS_D_AEQ_DEPTH_S); + caps->default_aeq_arm_st = udma_get_field(resp_d->arm_st_aeq_depth, + QUERY_PF_CAPS_D_AEQ_ARM_ST_M, + QUERY_PF_CAPS_D_AEQ_ARM_ST_S); + caps->default_ceq_arm_st = udma_get_field(resp_d->arm_st_aeq_depth, + QUERY_PF_CAPS_D_CEQ_ARM_ST_M, + QUERY_PF_CAPS_D_CEQ_ARM_ST_S); + caps->reserved_pds = udma_get_field(resp_d->num_uars_rsv_pds, + QUERY_PF_CAPS_D_RSV_PDS_M, + QUERY_PF_CAPS_D_RSV_PDS_S); + caps->num_uars = 1 << udma_get_field(resp_d->num_uars_rsv_pds, + QUERY_PF_CAPS_D_NUM_UARS_M, + QUERY_PF_CAPS_D_NUM_UARS_S); + caps->reserved_qps = udma_get_field(resp_d->rsv_uars_rsv_qps, + QUERY_PF_CAPS_D_RSV_QPS_M, + QUERY_PF_CAPS_D_RSV_QPS_S); + caps->reserved_uars = udma_get_field(resp_d->rsv_uars_rsv_qps, + QUERY_PF_CAPS_D_RSV_UARS_M, + QUERY_PF_CAPS_D_RSV_UARS_S); + caps->reserved_mrws = udma_get_field(resp_e->chunk_size_shift_rsv_mrws, + QUERY_PF_CAPS_E_RSV_MRWS_M, + QUERY_PF_CAPS_E_RSV_MRWS_S); + caps->chunk_sz = 1 << udma_get_field(resp_e->chunk_size_shift_rsv_mrws, + QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_M, + QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_S); + caps->reserved_cqs = udma_get_field(resp_e->rsv_cqs, + QUERY_PF_CAPS_E_RSV_CQS_M, + QUERY_PF_CAPS_E_RSV_CQS_S); + caps->reserved_srqs = udma_get_field(resp_e->rsv_srqs, + QUERY_PF_CAPS_E_RSV_SRQS_M, + QUERY_PF_CAPS_E_RSV_SRQS_S); + caps->reserved_lkey = udma_get_field(resp_e->rsv_lkey, + QUERY_PF_CAPS_E_RSV_LKEYS_M, + QUERY_PF_CAPS_E_RSV_LKEYS_S); + caps->default_ceq_max_cnt = le16_to_cpu(resp_e->ceq_max_cnt); + caps->default_ceq_period = le16_to_cpu(resp_e->ceq_period); + caps->default_aeq_max_cnt = le16_to_cpu(resp_e->aeq_max_cnt); + caps->default_aeq_period = le16_to_cpu(resp_e->aeq_period); + + caps->qpc_hop_num = ctx_hop_num; + caps->sccc_hop_num = ctx_hop_num; + caps->srqc_hop_num = ctx_hop_num; + caps->cqc_hop_num = ctx_hop_num; + caps->mpt_hop_num = ctx_hop_num; + caps->mtt_hop_num = pbl_hop_num; + caps->cqe_hop_num = pbl_hop_num; + caps->srqwqe_hop_num = pbl_hop_num; + caps->idx_hop_num = pbl_hop_num; + caps->wqe_sq_hop_num = udma_get_field(resp_d->wq_hop_num_max_srqs, + QUERY_PF_CAPS_D_SQWQE_HOP_NUM_M, + QUERY_PF_CAPS_D_SQWQE_HOP_NUM_S); + caps->wqe_sge_hop_num = udma_get_field(resp_d->wq_hop_num_max_srqs, + QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_M, + QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_S); + caps->wqe_rq_hop_num = udma_get_field(resp_d->wq_hop_num_max_srqs, + QUERY_PF_CAPS_D_RQWQE_HOP_NUM_M, + QUERY_PF_CAPS_D_RQWQE_HOP_NUM_S); + + return 0; +} + +static int udma_pf_profile(struct udma_dev *udma_dev) +{ + struct device *dev = udma_dev->dev; + int ret; + + ret = udma_query_func_info(udma_dev); + if (ret) { + dev_err(dev, "failed to query func info, ret = %d.\n", ret); + return ret; + } + + ret = udma_config_global_param(udma_dev); + if (ret) { + dev_err(dev, "failed to config global param, ret = %d.\n", ret); + return ret; + } + + ret = udma_set_vf_switch_param(udma_dev); + if (ret) { + dev_err(dev, "failed to set switch param, ret = %d.\n", ret); + return ret; + } + + ret = udma_query_caps(udma_dev); + if (ret) { + dev_err(dev, "failed to query caps, ret = %d.\n", ret); + return ret; + } + + return 0; +} + +static int udma_profile(struct udma_dev *udma_dev) +{ + struct device *dev = udma_dev->dev; + int ret; + + ret = udma_cmq_query_hw_info(udma_dev); + if (ret) { + dev_err(dev, "failed to query hardware info, ret = %d.\n", ret); + return ret; + } + + ret = udma_query_fw_ver(udma_dev); + if (ret) { + dev_err(dev, "failed to query firmware info, ret = %d.\n", ret); + return ret; + } + + udma_dev->sys_image_guid = be64_to_cpu(udma_dev->ub_dev.attr.guid); + + return udma_pf_profile(udma_dev); +} + +static int udma_cmq_init(struct udma_dev *udma_dev) +{ + return 0; +} + +static void udma_cmq_exit(struct udma_dev *udma_dev) +{ + +} + +static int udma_hw_init(struct udma_dev *udma_dev) +{ + return 0; +} + +static void udma_hw_exit(struct udma_dev *udma_dev) +{ + +} + +static const struct udma_hw udma_hw = { + .cmq_init = udma_cmq_init, + .cmq_exit = udma_cmq_exit, + .hw_profile = udma_profile, + .hw_init = udma_hw_init, + .hw_exit = udma_hw_exit, +}; + +static void udma_get_cfg(struct udma_dev *udma_dev, + struct hnae3_handle *handle) +{ + struct udma_priv *priv = (struct udma_priv *)udma_dev->priv; + int i; + + udma_dev->pci_dev = handle->pdev; + udma_dev->dev = &handle->pdev->dev; + udma_dev->hw = &udma_hw; + + /* Get info from NIC driver. */ + udma_dev->reg_base = handle->udmainfo.udma_io_base; + udma_dev->caps.num_ports = 1; + udma_dev->uboe.netdevs[0] = handle->udmainfo.netdev; + + for (i = 0; i < handle->udmainfo.num_vectors; i++) + udma_dev->irq[i] = pci_irq_vector(handle->pdev, + i + + handle->udmainfo.base_vector); + + /* cmd issue mode: 0 is poll, 1 is event */ + udma_dev->cmd_mod = 0; + + udma_dev->reset_cnt = handle->ae_algo->ops->ae_dev_reset_cnt(handle); + priv->handle = handle; +} + +static int __udma_init_instance(struct hnae3_handle *handle) +{ + struct udma_dev *udma_dev; + int ret; + + udma_dev = kzalloc(sizeof(*udma_dev), GFP_KERNEL); + if (ZERO_OR_NULL_PTR(udma_dev)) + return -ENOMEM; + + udma_dev->priv = kzalloc(sizeof(struct udma_priv), GFP_KERNEL); + if (ZERO_OR_NULL_PTR(udma_dev->priv)) { + ret = -ENOMEM; + goto error_failed_kzalloc; + } + + udma_get_cfg(udma_dev, handle); + + ret = udma_hnae_client_init(udma_dev); + if (ret) { + dev_err(udma_dev->dev, "UDMA Engine init failed(%d)!\n", ret); + goto error_failed_get_cfg; + } + handle->priv = udma_dev; + + return 0; +error_failed_get_cfg: + kfree(udma_dev->priv); +error_failed_kzalloc: + kfree(udma_dev); + + return ret; +} + +static void __udma_uninit_instance(struct hnae3_handle *handle, + bool reset) +{ + struct udma_dev *udma_dev = handle->priv; + + if (!udma_dev) + return; + + handle->priv = NULL; + + udma_hnae_client_exit(udma_dev); + kfree(udma_dev->priv); + kfree(udma_dev); +} + +static int udma_init_instance(struct hnae3_handle *handle) +{ + struct device *dev = &handle->pdev->dev; + const struct pci_device_id *id; + int ret; + + handle->udmainfo.instance_state = UDMA_STATE_INIT; + + id = pci_match_id(udma_hw_pci_tbl, handle->pdev); + if (!id) + return 0; + + ret = __udma_init_instance(handle); + if (ret) { + handle->udmainfo.instance_state = UDMA_STATE_NON_INIT; + if (ret == -EOPNOTSUPP) + return ret; + + dev_err(dev, "UDMA instance init failed! ret = %d\n", ret); + } + + handle->udmainfo.instance_state = UDMA_STATE_INITED; + + return 0; +} + +static void udma_uninit_instance(struct hnae3_handle *handle, bool reset) +{ + if (handle->udmainfo.instance_state != UDMA_STATE_INITED) + return; + + handle->udmainfo.instance_state = UDMA_STATE_UNINIT; + + __udma_uninit_instance(handle, reset); + + handle->udmainfo.instance_state = UDMA_STATE_NON_INIT; +} + +static const struct hnae3_client_ops udma_ops = { + .init_instance = udma_init_instance, + .uninit_instance = udma_uninit_instance, +}; + +static struct hnae3_client udma_client = { + .name = "udma", + .type = HNAE3_CLIENT_UDMA, + .ops = &udma_ops, +}; + +static int __init udma_init(void) +{ + return hnae3_register_client(&udma_client); +} + +static void __exit udma_exit(void) +{ + hnae3_unregister_client(&udma_client); +} + +module_init(udma_init); +module_exit(udma_exit); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_DESCRIPTION("UBUS UDMA Driver"); diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.h b/drivers/ub/hw/hns3/hns3_udma_hw.h new file mode 100644 index 000000000000..2f232a264f45 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_hw.h @@ -0,0 +1,263 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_HW_H +#define _UDMA_HW_H + +#include "urma/ubcore_types.h" + +struct udma_query_version { + uint16_t udma_vendor_id; + uint16_t udma_hw_version; + uint32_t rsv[5]; +}; + +struct udma_query_fw_info { + uint32_t fw_ver; + uint32_t rsv[5]; +}; + + +struct udma_pf_func_info { + uint32_t own_func_num; + uint32_t own_mac_id; + uint32_t rsv[4]; +}; + +struct udma_cmq_req { + uint32_t data[6]; +}; + +struct udma_vf_switch { + uint32_t udma_sel; + uint32_t fun_id; + uint32_t cfg; + uint32_t resv1; + uint32_t resv2; + uint32_t resv3; +}; + +struct udma_hw_id_query_cmq { + uint8_t chip_id; + uint8_t die_id; + uint8_t mac_id; + uint8_t reserved; + uint32_t func_id; + uint32_t rsv[4]; +}; + +#define NIC_ICL_SWITCH_CMD_UDMA_SEL_SHIFT 0 +#define NIC_ICL_SWITCH_CMD_UDMA_SEL BIT(NIC_ICL_SWITCH_CMD_UDMA_SEL_SHIFT) + +#define VF_SWITCH_DATA_FUN_ID_VF_ID_S 3 +#define VF_SWITCH_DATA_FUN_ID_VF_ID_M GENMASK(10, 3) + +#define VF_SWITCH_DATA_CFG_ALW_LPBK_S 1 +#define VF_SWITCH_DATA_CFG_ALW_LCL_LPBK_S 2 +#define VF_SWITCH_DATA_CFG_ALW_DST_OVRD_S 3 + +#define CMQ_REQ_FIELD_LOC(h, l) ((uint64_t)(h) << 32 | (l)) + +/* Fields of UDMA_OPC_CFG_GLOBAL_PARAM */ +#define CFG_GLOBAL_PARAM_1US_CYCLES CMQ_REQ_FIELD_LOC(9, 0) +#define CFG_GLOBAL_PARAM_UDP_PORT CMQ_REQ_FIELD_LOC(31, 16) + +#define UDMA_1US_CFG 999 + +enum { + UDMA_CMD_FLAG_IN = BIT(0), + UDMA_CMD_FLAG_NEXT = BIT(2), + UDMA_CMD_FLAG_WR = BIT(3), + UDMA_CMD_FLAG_NO_INTR = BIT(4), +}; + +enum udma_cmd_return_status { + CMD_EXEC_SUCCESS = 0, + CMD_NOT_EXIST = 2, +}; + +/* CMQ command */ +enum udma_opcode_type { + UDMA_QUERY_FW_VER = 0x0001, + UDMA_OPC_QUERY_HW_ID = 0x7032, + UDMA_OPC_QUERY_HW_VER = 0x8000, + UDMA_OPC_CFG_GLOBAL_PARAM = 0x8001, + UDMA_OPC_QUERY_FUNC_INFO = 0x8407, + UDMA_OPC_QUERY_PF_CAPS_NUM = 0x8408, + UDMA_OPC_QUERY_MB_ST = 0x8505, + UDMA_SWITCH_PARAMETER_CFG = 0x1033, +}; + +#define UDMA_QUERY_PF_CAPS_CMD_NUM 5 +struct udma_query_pf_caps_a { + uint8_t number_ports; + uint8_t local_ca_ack_delay; + uint16_t max_sq_sg; + uint16_t max_sq_inline; + uint16_t max_rq_sg; + uint32_t max_extend_sg; + uint16_t num_qpc_timer; + uint16_t num_cqc_timer; + uint16_t max_srq_sges; + uint8_t num_aeq_vectors; + uint8_t num_other_vectors; + uint8_t max_sq_desc_sz; + uint8_t max_rq_desc_sz; + uint8_t max_srq_desc_sz; + uint8_t cqe_sz; +}; + +struct udma_query_pf_caps_b { + uint8_t mtpt_entry_sz; + uint8_t irrl_entry_sz; + uint8_t trrl_entry_sz; + uint8_t cqc_entry_sz; + uint8_t srqc_entry_sz; + uint8_t idx_entry_sz; + uint8_t sccc_sz; + uint8_t max_mtu; + uint16_t qpc_sz; + uint16_t qpc_timer_entry_sz; + uint16_t cqc_timer_entry_sz; + uint8_t min_cqes; + uint8_t min_wqes; + uint32_t page_size_cap; + uint8_t pkey_table_len; + uint8_t phy_num_uars; + uint8_t ctx_hop_num; + uint8_t pbl_hop_num; +}; + +struct udma_query_pf_caps_c { + uint32_t cap_flags_num_pds; + uint32_t max_gid_num_cqs; + uint32_t cq_depth; + uint32_t num_mrws; + uint32_t ord_num_qps; + uint16_t sq_depth; + uint16_t rq_depth; +}; + +struct udma_query_pf_caps_d { + uint32_t wq_hop_num_max_srqs; + uint16_t srq_depth; + uint16_t cap_flags_ex; + uint32_t num_ceqs_ceq_depth; + uint32_t arm_st_aeq_depth; + uint32_t num_uars_rsv_pds; + uint32_t rsv_uars_rsv_qps; +}; + +struct udma_query_pf_caps_e { + uint32_t chunk_size_shift_rsv_mrws; + uint32_t rsv_cqs; + uint32_t rsv_srqs; + uint32_t rsv_lkey; + uint16_t ceq_max_cnt; + uint16_t ceq_period; + uint16_t aeq_max_cnt; + uint16_t aeq_period; +}; + +#define QUERY_PF_CAPS_C_NUM_PDS_S 0 +#define QUERY_PF_CAPS_C_NUM_PDS_M GENMASK(19, 0) + +#define QUERY_PF_CAPS_C_CAP_FLAGS_S 20 +#define QUERY_PF_CAPS_C_CAP_FLAGS_M GENMASK(31, 20) + +#define QUERY_PF_CAPS_C_NUM_CQS_S 0 +#define QUERY_PF_CAPS_C_NUM_CQS_M GENMASK(19, 0) + +#define QUERY_PF_CAPS_C_MAX_GID_S 20 +#define QUERY_PF_CAPS_C_MAX_GID_M GENMASK(28, 20) + +#define QUERY_PF_CAPS_C_CQ_DEPTH_S 0 +#define QUERY_PF_CAPS_C_CQ_DEPTH_M GENMASK(22, 0) + +#define QUERY_PF_CAPS_C_NUM_MRWS_S 0 +#define QUERY_PF_CAPS_C_NUM_MRWS_M GENMASK(19, 0) + +#define QUERY_PF_CAPS_C_NUM_QPS_S 0 +#define QUERY_PF_CAPS_C_NUM_QPS_M GENMASK(19, 0) + +#define QUERY_PF_CAPS_C_MAX_ORD_S 20 +#define QUERY_PF_CAPS_C_MAX_ORD_M GENMASK(27, 20) + +#define QUERY_PF_CAPS_D_NUM_SRQS_S 0 +#define QUERY_PF_CAPS_D_NUM_SRQS_M GENMASK(19, 0) + +#define QUERY_PF_CAPS_D_RQWQE_HOP_NUM_S 20 +#define QUERY_PF_CAPS_D_RQWQE_HOP_NUM_M GENMASK(21, 20) + +#define QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_S 22 +#define QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_M GENMASK(23, 22) + +#define QUERY_PF_CAPS_D_SQWQE_HOP_NUM_S 24 +#define QUERY_PF_CAPS_D_SQWQE_HOP_NUM_M GENMASK(25, 24) + +#define QUERY_PF_CAPS_D_CONG_TYPE_S 26 +#define QUERY_PF_CAPS_D_CONG_TYPE_M GENMASK(29, 26) + +/* default cap configuration for vf only */ +#define DEFAULT_AEQ_ARM_ST 0x3 +#define DEFAULT_CEQ_ARM_ST 0x3 +#define DEFAULT_CEQ_MAX_CNT 0x1 +#define DEFAULT_CEQ_PERIOD 0x10 +#define DEFAULT_AEQ_MAX_CNT 0x1 +#define DEFAULT_AEQ_PERIOD 0x10 + +#define QUERY_PF_CAPS_D_CEQ_DEPTH_S 0 +#define QUERY_PF_CAPS_D_CEQ_DEPTH_M GENMASK(21, 0) + +#define QUERY_PF_CAPS_D_NUM_CEQS_S 22 +#define QUERY_PF_CAPS_D_NUM_CEQS_M GENMASK(31, 22) + +#define QUERY_PF_CAPS_D_AEQ_DEPTH_S 0 +#define QUERY_PF_CAPS_D_AEQ_DEPTH_M GENMASK(21, 0) + +#define QUERY_PF_CAPS_D_AEQ_ARM_ST_S 22 +#define QUERY_PF_CAPS_D_AEQ_ARM_ST_M GENMASK(23, 22) + +#define QUERY_PF_CAPS_D_CEQ_ARM_ST_S 24 +#define QUERY_PF_CAPS_D_CEQ_ARM_ST_M GENMASK(25, 24) + +#define QUERY_PF_CAPS_D_RSV_PDS_S 0 +#define QUERY_PF_CAPS_D_RSV_PDS_M GENMASK(19, 0) + +#define QUERY_PF_CAPS_D_NUM_UARS_S 20 +#define QUERY_PF_CAPS_D_NUM_UARS_M GENMASK(27, 20) + +#define QUERY_PF_CAPS_D_RSV_QPS_S 0 +#define QUERY_PF_CAPS_D_RSV_QPS_M GENMASK(19, 0) + +#define QUERY_PF_CAPS_D_RSV_UARS_S 20 +#define QUERY_PF_CAPS_D_RSV_UARS_M GENMASK(27, 20) + +#define QUERY_PF_CAPS_E_RSV_MRWS_S 0 +#define QUERY_PF_CAPS_E_RSV_MRWS_M GENMASK(19, 0) + +#define QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_S 20 +#define QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_M GENMASK(31, 20) + +#define QUERY_PF_CAPS_E_RSV_CQS_S 0 +#define QUERY_PF_CAPS_E_RSV_CQS_M GENMASK(19, 0) + +#define QUERY_PF_CAPS_E_RSV_SRQS_S 0 +#define QUERY_PF_CAPS_E_RSV_SRQS_M GENMASK(19, 0) + +#define QUERY_PF_CAPS_E_RSV_LKEYS_S 0 +#define QUERY_PF_CAPS_E_RSV_LKEYS_M GENMASK(19, 0) + +#endif /* _UDMA_HW_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c new file mode 100644 index 000000000000..3fb0e1f83b4d --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include "hns3_udma_device.h" +#include "urma/ubcore_api.h" +#include "hns3_udma_device.h" + +static struct ubcore_ops g_udma_dev_ops = { + +}; + +static void udma_set_devname(struct udma_dev *udma_dev, + struct ubcore_device *ub_dev) +{ + scnprintf(udma_dev->dev_name, UBCORE_MAX_DEV_NAME, "udma%d", + udma_dev->func_id); + dev_info(udma_dev->dev, "Set dev_name %s\n", udma_dev->dev_name); + strlcpy(ub_dev->dev_name, udma_dev->dev_name, UBCORE_MAX_DEV_NAME); +} + +static int udma_register_device(struct udma_dev *udma_dev) +{ + struct ubcore_device *ub_dev = NULL; + struct udma_netdev *uboe = NULL; + + ub_dev = &udma_dev->ub_dev; + uboe = &udma_dev->uboe; + spin_lock_init(&uboe->lock); + ub_dev->transport_type = UBCORE_TRANSPORT_IB; + ub_dev->ops = &g_udma_dev_ops; + ub_dev->dev.parent = udma_dev->dev; + ub_dev->dma_dev = ub_dev->dev.parent; + ub_dev->netdev = udma_dev->uboe.netdevs[0]; + scnprintf(ub_dev->ops->driver_name, UBCORE_MAX_DRIVER_NAME, "udma_v1"); + udma_set_devname(udma_dev, ub_dev); + ub_dev->num_comp_vectors = udma_dev->irq_num; + + return ubcore_register_device(ub_dev); +} + +static void udma_unregister_device(struct udma_dev *udma_dev) +{ + struct ubcore_device *ub_dev = &udma_dev->ub_dev; + + ubcore_unregister_device(ub_dev); +} + +int udma_hnae_client_init(struct udma_dev *udma_dev) +{ + struct device *dev = udma_dev->dev; + int ret; + + ret = udma_dev->hw->cmq_init(udma_dev); + if (ret) { + dev_err(dev, "Init UB Command Queue failed!\n"); + goto error_failed_cmq_init; + } + + ret = udma_dev->hw->hw_profile(udma_dev); + if (ret) { + dev_err(dev, "Get UB engine profile failed!\n"); + goto error_failed_hw_profile; + } + + ret = udma_dev->hw->hw_init(udma_dev); + if (ret) { + dev_err(dev, "hw_init failed!\n"); + goto error_failed_engine_init; + } + + ret = udma_register_device(udma_dev); + if (ret) { + dev_err(dev, "udma register device failed!\n"); + goto error_failed_register_device; + } + + return 0; + +error_failed_register_device: + udma_dev->hw->hw_exit(udma_dev); + +error_failed_engine_init: +error_failed_hw_profile: + udma_dev->hw->cmq_exit(udma_dev); + +error_failed_cmq_init: + return ret; +} + +void udma_hnae_client_exit(struct udma_dev *udma_dev) +{ + udma_unregister_device(udma_dev); + + if (udma_dev->hw->hw_exit) + udma_dev->hw->hw_exit(udma_dev); + if (udma_dev->hw->cmq_exit) + udma_dev->hw->cmq_exit(udma_dev); +} -- Gitee From 9c9b286984c6f6623edf2ec0367380432ff66646 Mon Sep 17 00:00:00 2001 From: Weibo Zhao Date: Mon, 7 Aug 2023 15:03:08 +0800 Subject: [PATCH 02/42] hns3 udma: cmd and mailbox for hns3-udma driver driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA ------------------------------------------------ This patch add feature of cmd and mailbox for hns3-udma driver. There are two mode of cmd: polling mode and event mode. Signed-off-by: Weibo Zhao --- drivers/ub/hw/hns3/hns3_udma_cmd.c | 296 ++++++++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_cmd.h | 45 ++++ drivers/ub/hw/hns3/hns3_udma_device.h | 30 +++ drivers/ub/hw/hns3/hns3_udma_hw.c | 79 ++++++- drivers/ub/hw/hns3/hns3_udma_main.c | 16 ++ 5 files changed, 465 insertions(+), 1 deletion(-) diff --git a/drivers/ub/hw/hns3/hns3_udma_cmd.c b/drivers/ub/hw/hns3/hns3_udma_cmd.c index 27a936e0b800..452ae6371f3f 100644 --- a/drivers/ub/hw/hns3/hns3_udma_cmd.c +++ b/drivers/ub/hw/hns3/hns3_udma_cmd.c @@ -43,6 +43,75 @@ void udma_cmd_cleanup(struct udma_dev *udma_dev) up_write(&udma_dev->cmd.udma_mb_rwsem); } +int udma_cmd_use_events(struct udma_dev *udma_dev) +{ + struct udma_cmdq *udma_cmd = &udma_dev->cmd; + int i; + + udma_cmd->context = (struct udma_cmd_context *) + kcalloc(udma_cmd->max_cmds, sizeof(*udma_cmd->context), + GFP_KERNEL); + if (!udma_cmd->context) + return -ENOMEM; + + for (i = 0; i < udma_cmd->max_cmds; ++i) { + udma_cmd->context[i].token = i; + udma_cmd->context[i].next = i + 1; + init_completion(&udma_cmd->context[i].done); + } + udma_cmd->context[udma_cmd->max_cmds - 1].next = 0; + udma_cmd->free_head = 0; + + sema_init(&udma_cmd->event_sem, udma_cmd->max_cmds); + spin_lock_init(&udma_cmd->ctx_lock); + + udma_cmd->use_events = 1; + + return 0; +} + +void udma_cmd_use_polling(struct udma_dev *udma_dev) +{ + struct udma_cmdq *udma_cmd = &udma_dev->cmd; + + kfree(udma_cmd->context); + udma_cmd->use_events = 0; +} + +struct udma_cmd_mailbox *udma_alloc_cmd_mailbox(struct udma_dev *dev) +{ + struct udma_cmd_mailbox *mailbox; + + mailbox = kzalloc(sizeof(*mailbox), GFP_KERNEL); + if (!mailbox) + return ERR_PTR(-ENOMEM); + + down_read(&dev->cmd.udma_mb_rwsem); + mailbox->buf = + dma_pool_zalloc(dev->cmd.pool, GFP_KERNEL, &mailbox->dma); + if (!mailbox->buf) { + up_read(&dev->cmd.udma_mb_rwsem); + + kfree(mailbox); + return ERR_PTR(-ENOMEM); + } + + return mailbox; +} + +void udma_free_cmd_mailbox(struct udma_dev *dev, + struct udma_cmd_mailbox *mailbox) +{ + if (!mailbox) + return; + + dma_pool_free(dev->cmd.pool, mailbox->buf, mailbox->dma); + + up_read(&dev->cmd.udma_mb_rwsem); + + kfree(mailbox); +} + static uint32_t udma_cmd_hw_reseted(struct udma_dev *dev, uint64_t instance_stage, uint64_t reset_stage) @@ -300,3 +369,230 @@ int udma_cmq_send(struct udma_dev *dev, struct udma_cmq_desc *desc, int num) return ret; } + +static int udma_wait_mbox_complete(struct udma_dev *dev, uint32_t timeout, + uint8_t *complete_status) +{ + struct udma_mbox_status *mb_st; + struct udma_cmq_desc desc; + unsigned long end; + int ret = -EBUSY; + uint32_t status; + bool busy; + + mb_st = (struct udma_mbox_status *)desc.data; + end = msecs_to_jiffies(timeout) + jiffies; + while (udma_chk_mbox_is_avail(dev, &busy)) { + status = 0; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_QUERY_MB_ST, + true); + ret = __udma_cmq_send(dev, &desc, 1); + if (!ret) { + status = le32_to_cpu(mb_st->mb_status_hw_run); + /* No pending message exists in UDMA mbox. */ + if (!(status & MB_ST_HW_RUN_M)) + break; + } else if (!udma_chk_mbox_is_avail(dev, &busy)) { + break; + } + + if (time_after(jiffies, end)) { + dev_err_ratelimited(dev->dev, + "failed to wait mbox status 0x%x\n", + status); + return -ETIMEDOUT; + } + + cond_resched(); + ret = -EBUSY; + } + + if (!ret) { + *complete_status = (uint8_t)(status & MB_ST_COMPLETE_M); + } else if (!udma_chk_mbox_is_avail(dev, &busy)) { + /* Ignore all errors if the mbox is unavailable. */ + ret = busy ? -EBUSY : 0; + *complete_status = MB_ST_COMPLETE_M; + } + + return ret; +} + +static int __udma_post_mbox(struct udma_dev *dev, struct udma_cmq_desc *desc, + uint16_t token, int vfid_event) +{ + struct udma_mbox *mb = (struct udma_mbox *)desc->data; + + mb->token_event_en = cpu_to_le32(vfid_event + << UDMA_MB_EVENT_EN_SHIFT | token); + + return udma_cmq_send(dev, desc, 1); +} + +int udma_post_mbox(struct udma_dev *dev, struct udma_cmq_desc *desc, + uint16_t token, int vfid_event) +{ + uint8_t status = 0; + int ret; + + /* Waiting for the mbox to be idle */ + ret = udma_wait_mbox_complete(dev, UDMA_GO_BIT_TIMEOUT_MSECS, + &status); + if (unlikely(ret)) { + dev_err_ratelimited(dev->dev, + "failed to check post mbox status = 0x%x, ret = %d.\n", + status, ret); + return ret; + } + + /* Post new message to mbox */ + ret = __udma_post_mbox(dev, desc, token, vfid_event); + if (ret) + dev_err_ratelimited(dev->dev, + "failed to post mailbox, ret = %d.\n", ret); + + return ret; +} + +int udma_poll_mbox_done(struct udma_dev *dev, uint32_t timeout) +{ + uint8_t status = 0; + int ret; + + ret = udma_wait_mbox_complete(dev, timeout, &status); + if (!ret) { + if (status != MB_ST_COMPLETE_SUCC) + return -EBUSY; + } else { + dev_err_ratelimited(dev->dev, + "failed to check mbox status = 0x%x, ret = %d.\n", + status, ret); + } + + return ret; +} + +static int udma_cmd_mbox_post_hw(struct udma_dev *dev, + struct udma_cmq_desc *desc, + uint16_t token, int vfid_event) +{ + return dev->hw->post_mbox(dev, desc, token, vfid_event); +} + +static int __udma_cmd_mbox_poll(struct udma_dev *dev, + struct udma_cmq_desc *desc, + uint32_t timeout, int vfid) +{ + int vfid_event = (vfid << 1); + int ret, op; + + op = le32_to_cpu(((struct udma_mbox *)desc->data)->cmd_tag) & 0xff; + ret = udma_cmd_mbox_post_hw(dev, desc, CMD_POLL_TOKEN, vfid_event); + if (ret) { + dev_err_ratelimited(dev->dev, + "failed to post mailbox %x in poll mode, ret = %d.\n", + op, ret); + return ret; + } + + return dev->hw->poll_mbox_done(dev, timeout); +} + +static int udma_cmd_mbox_poll(struct udma_dev *dev, struct udma_cmq_desc *desc, + uint32_t timeout, int vfid) +{ + int ret; + + down(&dev->cmd.poll_sem); + ret = __udma_cmd_mbox_poll(dev, desc, timeout, vfid); + up(&dev->cmd.poll_sem); + + return ret; +} + +void udma_cmd_event(struct udma_dev *udma_dev, uint16_t token, uint8_t status, + uint64_t out_param) +{ + struct udma_cmd_context *ctx = + &udma_dev->cmd.context[token % udma_dev->cmd.max_cmds]; + + if (unlikely(token != ctx->token)) { + dev_err_ratelimited(udma_dev->dev, + "[cmd] invalid ae token 0x%x, context token is 0x%x.\n", + token, ctx->token); + return; + } + + ctx->result = (status == CMD_RST_PRC_SUCCESS) ? 0 : (-EIO); + ctx->out_param = out_param; + complete(&ctx->done); +} + +static int __udma_cmd_mbox_wait(struct udma_dev *udma_dev, + struct udma_cmq_desc *desc, + uint32_t timeout, int vfid) +{ + struct udma_cmdq *cmd = &udma_dev->cmd; + int vfid_event = (vfid << 1) | 0x1; + struct device *dev = udma_dev->dev; + struct udma_cmd_context *context; + int ret, op; + + spin_lock(&cmd->ctx_lock); + do { + context = &cmd->context[cmd->free_head]; + cmd->free_head = context->next; + } while (context->busy); + context->token += cmd->max_cmds; + context->busy = 1; + spin_unlock(&cmd->ctx_lock); + + reinit_completion(&context->done); + + op = le32_to_cpu(((struct udma_mbox *)desc->data)->cmd_tag) & 0xff; + ret = udma_cmd_mbox_post_hw(udma_dev, desc, context->token, vfid_event); + if (ret) { + dev_err_ratelimited(dev, + "failed to post mailbox %x in event mode, ret = %d.\n", + op, ret); + goto out; + } + + if (!wait_for_completion_timeout(&context->done, + msecs_to_jiffies(timeout))) { + dev_err_ratelimited(dev, "[cmd]token 0x%x of mailbox 0x%x timeout.\n", + context->token, op); + ret = -EBUSY; + goto out; + } + + ret = context->result; + if (ret) + dev_err_ratelimited(dev, "[cmd]token 0x%x of mailbox 0x%x error %d\n", + context->token, op, ret); + +out: + context->busy = 0; + return ret; +} + +static int udma_cmd_mbox_wait(struct udma_dev *dev, struct udma_cmq_desc *desc, + uint32_t timeout, int vfid) +{ + int ret; + + down(&dev->cmd.event_sem); + ret = __udma_cmd_mbox_wait(dev, desc, timeout, vfid); + up(&dev->cmd.event_sem); + + return ret; +} + +int udma_cmd_mbox(struct udma_dev *dev, struct udma_cmq_desc *desc, + uint32_t timeout, int vfid) +{ + if (dev->cmd.use_events) + return udma_cmd_mbox_wait(dev, desc, timeout, vfid); + else + return udma_cmd_mbox_poll(dev, desc, timeout, vfid); +} diff --git a/drivers/ub/hw/hns3/hns3_udma_cmd.h b/drivers/ub/hw/hns3/hns3_udma_cmd.h index 793a989494cb..ec3b4ece0213 100644 --- a/drivers/ub/hw/hns3/hns3_udma_cmd.h +++ b/drivers/ub/hw/hns3/hns3_udma_cmd.h @@ -38,8 +38,53 @@ enum { CMD_RST_PRC_EBUSY, }; +struct udma_mbox { + uint32_t in_param_l; + uint32_t in_param_h; + uint32_t out_param_l; + uint32_t out_param_h; + uint32_t cmd_tag; + uint32_t token_event_en; +}; + +struct udma_mbox_status { + uint32_t mb_status_hw_run; + uint32_t rsv[5]; +}; + +#define UDMA_GO_BIT_TIMEOUT_MSECS 10000 + +#define MB_ST_HW_RUN_M BIT(31) +#define MB_ST_COMPLETE_M GENMASK(7, 0) + +#define MB_ST_COMPLETE_SUCC 1 +#define UDMA_MB_EVENT_EN_SHIFT 16 + +void dump_desc(struct udma_dev *dev, struct udma_cmq_desc *desc); +struct udma_cmd_mailbox *udma_alloc_cmd_mailbox(struct udma_dev *dev); +void udma_free_cmd_mailbox(struct udma_dev *dev, + struct udma_cmd_mailbox *mailbox); +int udma_post_mbox(struct udma_dev *dev, struct udma_cmq_desc *desc, + uint16_t token, int vfid_event); +int udma_poll_mbox_done(struct udma_dev *dev, uint32_t timeout); +bool udma_chk_mbox_is_avail(struct udma_dev *dev, bool *busy); void udma_cmq_setup_basic_desc(struct udma_cmq_desc *desc, enum udma_opcode_type opcode, bool is_read); int udma_cmq_send(struct udma_dev *dev, struct udma_cmq_desc *desc, int num); +int udma_cmd_mbox(struct udma_dev *dev, struct udma_cmq_desc *desc, + uint32_t timeout, int vfid); +void udma_cmd_event(struct udma_dev *udma_dev, uint16_t token, uint8_t status, + uint64_t out_param); +static inline void mbox_desc_init(struct udma_mbox *mb, uint64_t in_param, + uint64_t out_param, uint32_t in_modifier, + uint16_t op) +{ + mb->in_param_l = cpu_to_le32(in_param); + mb->in_param_h = cpu_to_le32(in_param >> 32); + mb->out_param_l = cpu_to_le32(out_param); + mb->out_param_h = cpu_to_le32(out_param >> 32); + mb->cmd_tag = cpu_to_le32(in_modifier << 8 | op); +} + #endif /* _UDMA_CMD_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_device.h b/drivers/ub/hw/hns3/hns3_udma_device.h index 9df2aba5d94a..41b3df449ce4 100644 --- a/drivers/ub/hw/hns3/hns3_udma_device.h +++ b/drivers/ub/hw/hns3/hns3_udma_device.h @@ -25,6 +25,14 @@ #define UDMA_INVALID_ID 0xffff #define UDMA_MAX_IRQ_NUM 128 #define UDMA_CAP_FLAGS_EX_SHIFT 12 + +#define UDMA_CMQ_TX_TIMEOUT 30000 +#define UDMA_CMQ_DESC_NUM_S 3 +#define UDMA_CMD_CSQ_DESC_NUM 1024 + +#define UDMA_TX_CMQ_BASEADDR_L_REG 0x07000 +#define UDMA_TX_CMQ_BASEADDR_H_REG 0x07004 +#define UDMA_TX_CMQ_DEPTH_REG 0x07008 #define UDMA_TX_CMQ_PI_REG 0x07010 #define UDMA_TX_CMQ_CI_REG 0x07014 @@ -42,9 +50,15 @@ enum udma_instance_state { UDMA_STATE_UNINIT, }; +enum { + TYPE_CSQ = 1 +}; + struct udma_cmd_context { struct completion done; + int result; int next; + uint64_t out_param; uint16_t token; uint16_t busy; }; @@ -89,6 +103,12 @@ struct udma_cmdq { struct rw_semaphore udma_mb_rwsem; enum udma_cmdq_state state; }; + +struct udma_cmd_mailbox { + void *buf; + dma_addr_t dma; +}; + struct udma_netdev { spinlock_t lock; struct net_device *netdevs[UDMA_MAX_PORTS]; @@ -116,6 +136,10 @@ struct udma_hw { int (*hw_profile)(struct udma_dev *udma_dev); int (*hw_init)(struct udma_dev *udma_dev); void (*hw_exit)(struct udma_dev *udma_dev); + int (*post_mbox)(struct udma_dev *udma_dev, struct udma_cmq_desc *desc, + uint16_t token, int event); + int (*poll_mbox_done)(struct udma_dev *udma_dev, + uint32_t timeout); }; struct udma_caps { @@ -291,6 +315,12 @@ struct udma_dev { uint32_t cong_algo_tmpl_id; }; +int udma_cmd_init(struct udma_dev *udma_dev); +void udma_cmd_cleanup(struct udma_dev *udma_dev); +int udma_cmd_use_events(struct udma_dev *udma_dev); +void udma_cmd_use_polling(struct udma_dev *udma_dev); +int udma_cmq_send(struct udma_dev *udma_dev, + struct udma_cmq_desc *desc, int num); int udma_hnae_client_init(struct udma_dev *udma_dev); void udma_hnae_client_exit(struct udma_dev *udma_dev); diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.c b/drivers/ub/hw/hns3/hns3_udma_hw.c index 7fcbe31c07b8..efcb933a79b9 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hw.c +++ b/drivers/ub/hw/hns3/hns3_udma_hw.c @@ -393,14 +393,89 @@ static int udma_profile(struct udma_dev *udma_dev) return udma_pf_profile(udma_dev); } -static int udma_cmq_init(struct udma_dev *udma_dev) +static int udma_alloc_cmq_desc(struct udma_dev *udma_dev, + struct udma_cmq_ring *ring) +{ + int size = ring->desc_num * sizeof(struct udma_cmq_desc); + + ring->desc = kzalloc(size, GFP_KERNEL); + if (!ring->desc) + return -ENOMEM; + + ring->desc_dma_addr = dma_map_single(udma_dev->dev, ring->desc, size, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(udma_dev->dev, ring->desc_dma_addr)) { + ring->desc_dma_addr = 0; + kfree(ring->desc); + ring->desc = NULL; + + dev_err_ratelimited(udma_dev->dev, + "failed to map cmq desc addr.\n"); + return -ENOMEM; + } + + return 0; +} + +static void udma_free_cmq_desc(struct udma_dev *udma_dev, + struct udma_cmq_ring *ring) { + dma_unmap_single(udma_dev->dev, ring->desc_dma_addr, + ring->desc_num * sizeof(struct udma_cmq_desc), + DMA_BIDIRECTIONAL); + + ring->desc_dma_addr = 0; + kfree(ring->desc); + ring->desc = NULL; +} + +static int init_csq(struct udma_dev *udma_dev, + struct udma_cmq_ring *csq) +{ + dma_addr_t dma; + int ret; + + csq->desc_num = UDMA_CMD_CSQ_DESC_NUM; + mutex_init(&csq->lock); + csq->flag = TYPE_CSQ; + csq->head = 0; + + ret = udma_alloc_cmq_desc(udma_dev, csq); + if (ret) + return ret; + + dma = csq->desc_dma_addr; + ub_write(udma_dev, UDMA_TX_CMQ_BASEADDR_L_REG, lower_32_bits(dma)); + ub_write(udma_dev, UDMA_TX_CMQ_BASEADDR_H_REG, upper_32_bits(dma)); + ub_write(udma_dev, UDMA_TX_CMQ_DEPTH_REG, + (uint32_t)csq->desc_num >> UDMA_CMQ_DESC_NUM_S); + + /* Make sure to write CI first and then PI */ + ub_write(udma_dev, UDMA_TX_CMQ_CI_REG, 0); + ub_write(udma_dev, UDMA_TX_CMQ_PI_REG, 0); + return 0; } +static int udma_cmq_init(struct udma_dev *udma_dev) +{ + struct udma_priv *priv = (struct udma_priv *)udma_dev->priv; + int ret; + + priv->cmq.tx_timeout = UDMA_CMQ_TX_TIMEOUT; + + ret = init_csq(udma_dev, &priv->cmq.csq); + if (ret) + dev_err(udma_dev->dev, "failed to init CSQ, ret = %d.\n", ret); + + return ret; +} + static void udma_cmq_exit(struct udma_dev *udma_dev) { + struct udma_priv *priv = (struct udma_priv *)udma_dev->priv; + udma_free_cmq_desc(udma_dev, &priv->cmq.csq); } static int udma_hw_init(struct udma_dev *udma_dev) @@ -419,6 +494,8 @@ static const struct udma_hw udma_hw = { .hw_profile = udma_profile, .hw_init = udma_hw_init, .hw_exit = udma_hw_exit, + .post_mbox = udma_post_mbox, + .poll_mbox_done = udma_poll_mbox_done, }; static void udma_get_cfg(struct udma_dev *udma_dev, diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c index 3fb0e1f83b4d..46ad4d75b628 100644 --- a/drivers/ub/hw/hns3/hns3_udma_main.c +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -74,6 +74,21 @@ int udma_hnae_client_init(struct udma_dev *udma_dev) goto error_failed_hw_profile; } + ret = udma_cmd_init(udma_dev); + if (ret) { + dev_err(dev, "cmd init failed!\n"); + goto error_failed_cmd_init; + } + + if (udma_dev->cmd_mod) { + ret = udma_cmd_use_events(udma_dev); + if (ret) { + udma_dev->cmd_mod = 0; + dev_warn(dev, + "Cmd event mode failed, set back to poll!\n"); + } + } + ret = udma_dev->hw->hw_init(udma_dev); if (ret) { dev_err(dev, "hw_init failed!\n"); @@ -92,6 +107,7 @@ int udma_hnae_client_init(struct udma_dev *udma_dev) udma_dev->hw->hw_exit(udma_dev); error_failed_engine_init: +error_failed_cmd_init: error_failed_hw_profile: udma_dev->hw->cmq_exit(udma_dev); -- Gitee From b330adf6e87d21bbb22a767093a259d0440a7496 Mon Sep 17 00:00:00 2001 From: zhaoweibo Date: Tue, 17 Oct 2023 14:28:32 +0800 Subject: [PATCH 03/42] hns3 udma: function of hardware init driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA ------------------------------------------------ Ex_doorbell link table, hardware_entry_management table and timer_linktable should be set in function of hardware init. Ex_doorbell link table and timer_linktable are created by driver but only used by hardware. Hardware_entry_management(HEM) table is the size of resources like qps. Signed-off-by: Weibo Zhao --- drivers/ub/hw/hns3/Makefile | 3 +- drivers/ub/hw/hns3/hns3_udma_cmd.h | 80 ++- drivers/ub/hw/hns3/hns3_udma_common.h | 19 + drivers/ub/hw/hns3/hns3_udma_device.h | 179 ++++- drivers/ub/hw/hns3/hns3_udma_hem.c | 923 ++++++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_hem.h | 121 +++- drivers/ub/hw/hns3/hns3_udma_hw.c | 967 +++++++++++++++++++++++++- drivers/ub/hw/hns3/hns3_udma_hw.h | 130 ++++ drivers/ub/hw/hns3/hns3_udma_main.c | 206 +++++- 9 files changed, 2622 insertions(+), 6 deletions(-) create mode 100644 drivers/ub/hw/hns3/hns3_udma_hem.c diff --git a/drivers/ub/hw/hns3/Makefile b/drivers/ub/hw/hns3/Makefile index bcd8cc619962..122bb55bd26c 100644 --- a/drivers/ub/hw/hns3/Makefile +++ b/drivers/ub/hw/hns3/Makefile @@ -7,6 +7,7 @@ MODULE_NAME := hns3_udma ccflags-y += -I$(srctree)/drivers/net/ethernet/hisilicon/hns3/ \ -I$(srctree)/drivers/ub/urma/ -$(MODULE_NAME)-objs := hns3_udma_hw.o hns3_udma_main.o hns3_udma_cmd.o +$(MODULE_NAME)-objs := hns3_udma_hw.o hns3_udma_main.o hns3_udma_cmd.o \ + hns3_udma_hem.o obj-$(CONFIG_UB_UDMA_HNS3) := hns3_udma.o diff --git a/drivers/ub/hw/hns3/hns3_udma_cmd.h b/drivers/ub/hw/hns3/hns3_udma_cmd.h index ec3b4ece0213..ece4ed394456 100644 --- a/drivers/ub/hw/hns3/hns3_udma_cmd.h +++ b/drivers/ub/hw/hns3/hns3_udma_cmd.h @@ -25,11 +25,89 @@ #define SUB_OPCODE_IDX 4 enum { + /* QPC BT commands */ + UDMA_CMD_WRITE_QPC_BT0 = 0x0, + UDMA_CMD_WRITE_QPC_BT1 = 0x1, + UDMA_CMD_WRITE_QPC_BT2 = 0x2, + UDMA_CMD_READ_QPC_BT0 = 0x4, + UDMA_CMD_READ_QPC_BT1 = 0x5, + UDMA_CMD_READ_QPC_BT2 = 0x6, + UDMA_CMD_DESTROY_QPC_BT0 = 0x8, + UDMA_CMD_DESTROY_QPC_BT1 = 0x9, + UDMA_CMD_DESTROY_QPC_BT2 = 0xa, + + /* QPC operation */ + UDMA_CMD_MODIFY_QPC = 0x41, + UDMA_CMD_QUERY_QPC = 0x42, + + UDMA_CMD_MODIFY_CQC = 0x52, + UDMA_CMD_QUERY_CQC = 0x53, + /* CQC BT commands */ + UDMA_CMD_WRITE_CQC_BT0 = 0x10, + UDMA_CMD_WRITE_CQC_BT1 = 0x11, + UDMA_CMD_WRITE_CQC_BT2 = 0x12, + UDMA_CMD_READ_CQC_BT0 = 0x14, + UDMA_CMD_READ_CQC_BT1 = 0x15, + UDMA_CMD_READ_CQC_BT2 = 0x1b, + UDMA_CMD_DESTROY_CQC_BT0 = 0x18, + UDMA_CMD_DESTROY_CQC_BT1 = 0x19, + UDMA_CMD_DESTROY_CQC_BT2 = 0x1a, + + /* MPT BT commands */ + UDMA_CMD_WRITE_MPT_BT0 = 0x20, + UDMA_CMD_WRITE_MPT_BT1 = 0x21, + UDMA_CMD_WRITE_MPT_BT2 = 0x22, + UDMA_CMD_READ_MPT_BT0 = 0x24, + UDMA_CMD_READ_MPT_BT1 = 0x25, + UDMA_CMD_READ_MPT_BT2 = 0x26, + UDMA_CMD_DESTROY_MPT_BT0 = 0x28, + UDMA_CMD_DESTROY_MPT_BT1 = 0x29, + UDMA_CMD_DESTROY_MPT_BT2 = 0x2a, + /* CQC TIMER commands */ UDMA_CMD_WRITE_CQC_TIMER_BT0 = 0x23, + UDMA_CMD_READ_CQC_TIMER_BT0 = 0x27, + + /* MPT commands */ + UDMA_CMD_CREATE_MPT = 0xd, + UDMA_CMD_DESTROY_MPT = 0xf, + UDMA_CMD_QUERY_MPT = 0x62, + + /* SRQC BT commands */ + UDMA_CMD_WRITE_SRQC_BT0 = 0x30, + UDMA_CMD_WRITE_SRQC_BT1 = 0x31, + UDMA_CMD_WRITE_SRQC_BT2 = 0x32, + UDMA_CMD_READ_SRQC_BT0 = 0x34, + UDMA_CMD_READ_SRQC_BT1 = 0x35, + UDMA_CMD_READ_SRQC_BT2 = 0x36, + UDMA_CMD_DESTROY_SRQC_BT0 = 0x38, + UDMA_CMD_DESTROY_SRQC_BT1 = 0x39, + UDMA_CMD_DESTROY_SRQC_BT2 = 0x3a, /* QPC TIMER commands */ - UDMA_CMD_WRITE_QPC_TIMER_BT0 = 0x33, + UDMA_CMD_WRITE_QPC_TIMER_BT0 = 0x33, + UDMA_CMD_READ_QPC_TIMER_BT0 = 0x37, + + /* QP/EE commands */ + UDMA_CMD_CREATE_SRQ = 0x70, + UDMA_CMD_MODIFY_SRQC = 0x72, + UDMA_CMD_QUERY_SRQC = 0x73, + UDMA_CMD_DESTROY_SRQ = 0x74, + + /* EQC commands */ + UDMA_CMD_CREATE_AEQC = 0x80, + UDMA_CMD_DESTROY_AEQC = 0x83, + UDMA_CMD_CREATE_CEQC = 0x90, + UDMA_CMD_DESTROY_CEQC = 0x93, + + /* SCC CTX BT commands */ + UDMA_CMD_READ_SCCC_BT0 = 0xa4, + UDMA_CMD_WRITE_SCCC_BT0 = 0xa5, + + /* CQ commands */ + UDMA_CMD_CREATE_CQC = 0x16, + UDMA_CMD_DESTROY_CQC = 0x17, + UDMA_CMD_RESERVED = 0xff, }; enum { diff --git a/drivers/ub/hw/hns3/hns3_udma_common.h b/drivers/ub/hw/hns3/hns3_udma_common.h index e726c550ba1f..ec6bc584c640 100644 --- a/drivers/ub/hw/hns3/hns3_udma_common.h +++ b/drivers/ub/hw/hns3/hns3_udma_common.h @@ -35,6 +35,15 @@ #define udma_set_bit(origin, shift, val) \ udma_set_field((origin), (1ul << (shift)), (shift), (val)) +#define _udma_reg_enable(ptr, field) \ + ({ \ + const uint32_t *_ptr = (uint32_t *)(ptr); \ + *((uint32_t *)_ptr + ((field) >> 32) / 32) |= cpu_to_le32( \ + BIT((((field) << 32) >> 32) % 32)); \ + }) + +#define udma_reg_enable(ptr, field) _udma_reg_enable(ptr, field) + #define _udma_reg_clear(ptr, field) \ ({ \ const uint32_t *_ptr = (uint32_t *)(ptr); \ @@ -56,4 +65,14 @@ #define udma_reg_write(ptr, field, val) _udma_reg_write(ptr, field, val) +#define _udma_reg_read(ptr, field) \ + ({ \ + const uint32_t *_ptr = (uint32_t *)(ptr); \ + BUILD_BUG_ON((((field) >> 32) / 32) != ((((field) << 32) >> 32) / 32)); \ + FIELD_GET(GENMASK(((field) >> 32) % 32, (((field) << 32) >> 32) % 32), \ + le32_to_cpu(*((uint32_t *)_ptr + ((field) >> 32) / 32))); \ + }) + +#define udma_reg_read(ptr, field) _udma_reg_read(ptr, field) + #endif /* _UDMA_COMMON_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_device.h b/drivers/ub/hw/hns3/hns3_udma_device.h index 41b3df449ce4..d668e55bde14 100644 --- a/drivers/ub/hw/hns3/hns3_udma_device.h +++ b/drivers/ub/hw/hns3/hns3_udma_device.h @@ -22,8 +22,27 @@ #include "hns3_udma_common.h" #define UDMA_MAX_PORTS 6 + +#define BA_BYTE_LEN 8 + #define UDMA_INVALID_ID 0xffff + +/* Configure to HW for PAGE_SIZE larger than 4KB */ +#define PG_SHIFT_OFFSET (PAGE_SHIFT - 12) #define UDMA_MAX_IRQ_NUM 128 +#define UDMA_QP_BANK_NUM 8 +#define UDMA_QPC_SZ 512 +#define UDMA_CQE_SZ 64 +#define UDMA_SCCC_SZ 64 +#define UDMA_GMV_ENTRY_SZ 32 + +#define UDMA_CQ_BANK_NUM 4 +/* The minimum page size is 4K for hardware */ +#define UDMA_HW_PAGE_SHIFT 12 +#define UDMA_PAGE_SIZE (1 << UDMA_HW_PAGE_SHIFT) +#define udma_hw_page_align(x) ALIGN(x, 1 << UDMA_HW_PAGE_SHIFT) + +#define UDMA_HOP_NUM_0 0xff #define UDMA_CAP_FLAGS_EX_SHIFT 12 #define UDMA_CMQ_TX_TIMEOUT 30000 @@ -50,10 +69,59 @@ enum udma_instance_state { UDMA_STATE_UNINIT, }; + +enum { + /* discard BIT(2), reserved for compatibility */ + UDMA_CAP_FLAG_CQ_RECORD_DB = BIT(3), + UDMA_CAP_FLAG_QP_RECORD_DB = BIT(4), + UDMA_CAP_FLAG_SRQ = BIT(5), + UDMA_CAP_FLAG_QP_FLOW_CTRL = BIT(9), + UDMA_CAP_FLAG_DIRECT_WQE = BIT(12), + UDMA_CAP_FLAG_SDI_MODE = BIT(14), + UDMA_CAP_FLAG_DCA_MODE = BIT(15), + UDMA_CAP_FLAG_WRITE_NOTIFY = BIT(16), + UDMA_CAP_FLAG_STASH = BIT(17), + UDMA_CAP_FLAG_CQE_INLINE = BIT(19), + UDMA_CAP_FLAG_SRQ_RECORD_DB = BIT(22), + UDMA_CAP_FLAG_OOR = BIT(25), + UDMA_CAP_FLAG_AR = BIT(26), + UDMA_CAP_FLAG_POE = BIT(27), +}; + enum { TYPE_CSQ = 1 }; + +enum { + UDMA_BUF_DIRECT = BIT(0), + UDMA_BUF_NOSLEEP = BIT(1), + UDMA_BUF_NOFAIL = BIT(2), +}; + +struct udma_ida { + struct ida ida; + uint32_t min; /* Lowest ID to allocate. */ + uint32_t max; /* Highest ID to allocate. */ +}; +struct udma_buf_list { + void *buf; + dma_addr_t map; +}; + +struct udma_buf { + struct udma_buf_list *trunk_list; + uint32_t ntrunks; + uint32_t npages; + uint32_t trunk_shift; + uint32_t page_shift; +}; + +struct udma_link_table { + struct udma_buf_list table; + struct udma_buf *buf; +}; +struct udma_dev; struct udma_cmd_context { struct completion done; int result; @@ -127,9 +195,25 @@ struct udma_cmq { struct udma_priv { struct hnae3_handle *handle; struct udma_cmq cmq; + struct udma_link_table ext_llm; +}; + +struct udma_hem_table { + /* HEM type: 0 = qpc, 1 = mtt, 2 = cqc, 3 = srq, 4 = other */ + uint32_t type; + /* HEM array elment num */ + uint64_t num_hem; + /* Single obj size */ + uint64_t obj_size; + uint64_t table_chunk_size; + struct mutex mutex; + struct udma_hem **hem; + uint64_t **bt_l1; + dma_addr_t *bt_l1_dma_addr; + uint64_t **bt_l0; + dma_addr_t *bt_l0_dma_addr; }; -struct udma_dev; struct udma_hw { int (*cmq_init)(struct udma_dev *udma_dev); void (*cmq_exit)(struct udma_dev *udma_dev); @@ -140,6 +224,11 @@ struct udma_hw { uint16_t token, int event); int (*poll_mbox_done)(struct udma_dev *udma_dev, uint32_t timeout); + int (*set_hem)(struct udma_dev *udma_dev, + struct udma_hem_table *table, int obj, int step_idx); + int (*clear_hem)(struct udma_dev *udma_dev, + struct udma_hem_table *table, int obj, + int step_idx); }; struct udma_caps { @@ -290,6 +379,54 @@ struct udma_caps { uint8_t poe_ch_num; }; +struct udma_idx_table { + uint32_t *spare_idx; + uint32_t head; + uint32_t tail; +}; + +struct udma_bank { + struct ida ida; + uint32_t inuse; /* Number of IDs allocated */ + uint32_t min; /* Lowest ID to allocate. */ + uint32_t max; /* Highest ID to allocate. */ + uint32_t next; /* Next ID to allocate. */ +}; + +struct udma_qp_table { + struct xarray xa; + struct udma_hem_table qp_table; + struct udma_hem_table irrl_table; + struct udma_hem_table trrl_table; + struct udma_hem_table sccc_table; + struct udma_bank bank[UDMA_QP_BANK_NUM]; + struct mutex bank_mutex; + struct udma_idx_table idx_table; +}; + +struct udma_jfc_table { + struct xarray xa; + struct udma_hem_table table; + struct udma_bank bank[UDMA_CQ_BANK_NUM]; + struct mutex bank_mutex; +}; + +struct udma_jfs_table { + struct xarray xa; + struct udma_ida jfs_ida; +}; + +struct udma_jfr_table { + struct xarray xa; + struct udma_hem_table table; + struct udma_ida jfr_ida; +}; + +struct udma_seg_table { + struct udma_ida seg_ida; + struct udma_hem_table table; +}; + struct udma_dev { struct ubcore_device ub_dev; struct pci_dev *pci_dev; @@ -313,8 +450,46 @@ struct udma_dev { uint16_t func_id; uint32_t func_num; uint32_t cong_algo_tmpl_id; + struct udma_jfs_table jfs_table; + struct udma_jfr_table jfr_table; + struct udma_seg_table seg_table; + struct udma_jfc_table jfc_table; + struct udma_qp_table qp_table; + struct udma_hem_table qpc_timer_table; + struct udma_hem_table cqc_timer_table; + struct udma_hem_table gmv_table; }; +static inline uint64_t to_hr_hw_page_addr(uint64_t addr) +{ + return addr >> UDMA_HW_PAGE_SHIFT; +} + +static inline uint32_t to_hr_hw_page_shift(uint32_t page_shift) +{ + return page_shift - UDMA_HW_PAGE_SHIFT; +} + +static inline uint32_t to_udma_hem_hopnum(uint32_t hopnum, uint32_t count) +{ + if (count > 0) + return hopnum == UDMA_HOP_NUM_0 ? 0 : hopnum; + + return 0; +} + +static inline dma_addr_t udma_buf_dma_addr(struct udma_buf *buf, + uint32_t offset) +{ + return buf->trunk_list[offset >> buf->trunk_shift].map + + (offset & ((1 << buf->trunk_shift) - 1)); +} + +static inline dma_addr_t udma_buf_page(struct udma_buf *buf, uint32_t idx) +{ + return udma_buf_dma_addr(buf, idx << buf->page_shift); +} + int udma_cmd_init(struct udma_dev *udma_dev); void udma_cmd_cleanup(struct udma_dev *udma_dev); int udma_cmd_use_events(struct udma_dev *udma_dev); @@ -323,5 +498,7 @@ int udma_cmq_send(struct udma_dev *udma_dev, struct udma_cmq_desc *desc, int num); int udma_hnae_client_init(struct udma_dev *udma_dev); void udma_hnae_client_exit(struct udma_dev *udma_dev); +void udma_cleanup_common_hem(struct udma_dev *udma_dev); +int udma_init_common_hem(struct udma_dev *udma_dev); #endif /* _UDMA_DEVICE_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_hem.c b/drivers/ub/hw/hns3/hns3_udma_hem.c new file mode 100644 index 000000000000..34284c993b59 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_hem.c @@ -0,0 +1,923 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include "urma/ubcore_api.h" +#include "hns3_udma_device.h" +#include "hns3_udma_hem.h" + +bool udma_check_whether_mhop(struct udma_dev *udma_dev, uint32_t type) +{ + int hop_num = 0; + + switch (type) { + case HEM_TYPE_QPC: + hop_num = udma_dev->caps.qpc_hop_num; + break; + case HEM_TYPE_MTPT: + hop_num = udma_dev->caps.mpt_hop_num; + break; + case HEM_TYPE_CQC: + hop_num = udma_dev->caps.cqc_hop_num; + break; + case HEM_TYPE_SRQC: + hop_num = udma_dev->caps.srqc_hop_num; + break; + case HEM_TYPE_SCCC: + hop_num = udma_dev->caps.sccc_hop_num; + break; + case HEM_TYPE_QPC_TIMER: + hop_num = udma_dev->caps.qpc_timer_hop_num; + break; + case HEM_TYPE_CQC_TIMER: + hop_num = udma_dev->caps.cqc_timer_hop_num; + break; + case HEM_TYPE_GMV: + hop_num = udma_dev->caps.gmv_hop_num; + break; + default: + return false; + } + + return hop_num ? true : false; +} + +static bool udma_check_hem_null(struct udma_hem **hem, uint64_t hem_idx, + uint32_t bt_chunk_num, uint64_t hem_max_num) +{ + uint64_t start_idx = round_down(hem_idx, bt_chunk_num); + uint64_t check_max_num = start_idx + bt_chunk_num; + uint64_t i; + + for (i = start_idx; (i < check_max_num) && (i < hem_max_num); i++) + if (i != hem_idx && hem[i]) + return false; + + return true; +} + +static bool udma_check_bt_null(uint64_t **bt, uint64_t ba_idx, + uint32_t bt_chunk_num) +{ + uint64_t start_idx = round_down(ba_idx, bt_chunk_num); + uint32_t i; + + for (i = start_idx; i < bt_chunk_num; i++) + if (i != ba_idx && bt[i]) + return false; + + return true; +} + +static int udma_get_bt_num(uint32_t table_type, uint32_t hop_num) +{ + if (check_whether_bt_num_3(table_type, hop_num)) + return 3; + else if (check_whether_bt_num_2(table_type, hop_num)) + return 2; + else if (check_whether_bt_num_1(table_type, hop_num)) + return 1; + else + return 0; +} + +static int get_hem_table_config(struct udma_dev *udma_dev, + struct udma_hem_mhop *mhop, + uint32_t type) +{ + struct device *dev = udma_dev->dev; + + switch (type) { + case HEM_TYPE_QPC: + mhop->buf_chunk_size = 1 << (udma_dev->caps.qpc_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (udma_dev->caps.qpc_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = udma_dev->caps.qpc_bt_num; + mhop->hop_num = udma_dev->caps.qpc_hop_num; + break; + case HEM_TYPE_MTPT: + mhop->buf_chunk_size = 1 << (udma_dev->caps.mpt_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (udma_dev->caps.mpt_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = udma_dev->caps.mpt_bt_num; + mhop->hop_num = udma_dev->caps.mpt_hop_num; + break; + case HEM_TYPE_CQC: + mhop->buf_chunk_size = 1 << (udma_dev->caps.cqc_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (udma_dev->caps.cqc_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = udma_dev->caps.cqc_bt_num; + mhop->hop_num = udma_dev->caps.cqc_hop_num; + break; + case HEM_TYPE_SCCC: + mhop->buf_chunk_size = 1 << (udma_dev->caps.sccc_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (udma_dev->caps.sccc_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = udma_dev->caps.sccc_bt_num; + mhop->hop_num = udma_dev->caps.sccc_hop_num; + break; + case HEM_TYPE_QPC_TIMER: + mhop->buf_chunk_size = 1 << (udma_dev->caps.qpc_timer_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (udma_dev->caps.qpc_timer_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = udma_dev->caps.qpc_timer_bt_num; + mhop->hop_num = udma_dev->caps.qpc_timer_hop_num; + break; + case HEM_TYPE_CQC_TIMER: + mhop->buf_chunk_size = 1 << (udma_dev->caps.cqc_timer_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (udma_dev->caps.cqc_timer_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = udma_dev->caps.cqc_timer_bt_num; + mhop->hop_num = udma_dev->caps.cqc_timer_hop_num; + break; + case HEM_TYPE_SRQC: + mhop->buf_chunk_size = 1 << (udma_dev->caps.srqc_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (udma_dev->caps.srqc_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = udma_dev->caps.srqc_bt_num; + mhop->hop_num = udma_dev->caps.srqc_hop_num; + break; + case HEM_TYPE_GMV: + mhop->buf_chunk_size = 1 << (udma_dev->caps.gmv_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (udma_dev->caps.gmv_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = udma_dev->caps.gmv_bt_num; + mhop->hop_num = udma_dev->caps.gmv_hop_num; + break; + default: + dev_err(dev, "table %u not support multi-hop addressing!\n", + type); + return -EINVAL; + } + + return 0; +} + +int udma_calc_hem_mhop(struct udma_dev *udma_dev, + struct udma_hem_table *table, uint64_t *obj, + struct udma_hem_mhop *mhop) +{ + struct device *dev = udma_dev->dev; + uint32_t chunk_ba_num; + uint32_t chunk_size; + uint32_t table_idx; + uint32_t bt_num; + int ret; + + ret = get_hem_table_config(udma_dev, mhop, table->type); + if (ret) + return ret; + + if (!obj) + return 0; + + /* + * QPC/MTPT/CQC/SRQC/SCCC alloc hem for buffer pages. + * MTT/CQE alloc hem for bt pages. + */ + bt_num = udma_get_bt_num(table->type, mhop->hop_num); + chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; + chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : + mhop->bt_chunk_size; + table_idx = *obj / (chunk_size / table->obj_size); + switch (bt_num) { + case 3: + mhop->l2_idx = table_idx & (chunk_ba_num - 1); + mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1); + mhop->l0_idx = (table_idx / chunk_ba_num) / chunk_ba_num; + break; + case 2: + mhop->l1_idx = table_idx & (chunk_ba_num - 1); + mhop->l0_idx = table_idx / chunk_ba_num; + break; + case 1: + mhop->l0_idx = table_idx; + break; + default: + dev_err(dev, "table %u not support hop_num = %u!\n", + table->type, mhop->hop_num); + return -EINVAL; + } + if (mhop->l0_idx >= mhop->ba_l0_num) + mhop->l0_idx %= mhop->ba_l0_num; + + return 0; +} + +static void udma_free_hem(struct udma_dev *udma_dev, struct udma_hem *hem) +{ + struct udma_hem_chunk *chunk, *tmp; + int i; + + if (!hem) + return; + + list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) { + for (i = 0; i < chunk->npages; ++i) + dma_free_coherent(udma_dev->dev, + sg_dma_len(&chunk->mem[i]), + chunk->buf[i], + sg_dma_address(&chunk->mem[i])); + kfree(chunk); + } + + kfree(hem); +} + +static struct udma_hem *udma_alloc_hem(struct udma_dev *udma_dev, int npages, + uint64_t hem_alloc_size, gfp_t gfp_mask) +{ + struct udma_hem_chunk *chunk = NULL; + struct scatterlist *mem; + struct udma_hem *hem; + int pages = npages; + int order; + void *buf; + + hem = kmalloc(sizeof(*hem), gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); + if (!hem) + return NULL; + + refcount_set(&hem->refcount, 0); + INIT_LIST_HEAD(&hem->chunk_list); + + order = get_order(hem_alloc_size); + + chunk = kmalloc(sizeof(*chunk), gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); + if (!chunk) + goto fail; + + sg_init_table(chunk->mem, UDMA_HEM_CHUNK_LEN); + chunk->npages = 0; + chunk->nsg = 0; + memset(chunk->buf, 0, sizeof(chunk->buf)); + list_add_tail(&chunk->list, &hem->chunk_list); + while (pages > 0) { + while (1 << order > pages) + --order; + mem = &chunk->mem[chunk->npages]; + buf = dma_alloc_coherent(udma_dev->dev, PAGE_SIZE << order, + &sg_dma_address(mem), gfp_mask); + if (!buf) + goto fail; + + chunk->buf[chunk->npages] = buf; + sg_dma_len(mem) = PAGE_SIZE << order; + + ++chunk->npages; + ++chunk->nsg; + pages -= 1 << order; + } + + return hem; + +fail: + udma_free_hem(udma_dev, hem); + return NULL; +} + +static int calc_hem_config(struct udma_dev *udma_dev, + struct udma_hem_table *table, uint64_t obj, + struct udma_hem_mhop *mhop, + struct udma_hem_index *index) +{ + struct device *dev = udma_dev->dev; + uint32_t l0_idx, l1_idx, l2_idx; + uint64_t mhop_obj = obj; + uint32_t chunk_ba_num; + uint32_t bt_num; + int ret; + + ret = udma_calc_hem_mhop(udma_dev, table, &mhop_obj, mhop); + if (ret) + return ret; + + l0_idx = mhop->l0_idx; + l1_idx = mhop->l1_idx; + l2_idx = mhop->l2_idx; + chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; + bt_num = udma_get_bt_num(table->type, mhop->hop_num); + switch (bt_num) { + case 3: + index->l1 = l0_idx * chunk_ba_num + l1_idx; + index->l0 = l0_idx; + index->buf = l0_idx * chunk_ba_num * chunk_ba_num + + l1_idx * chunk_ba_num + l2_idx; + break; + case 2: + index->l0 = l0_idx; + index->buf = l0_idx * chunk_ba_num + l1_idx; + break; + case 1: + index->buf = l0_idx; + break; + default: + dev_err(dev, "table %u not support mhop.hop_num = %u!\n", + table->type, mhop->hop_num); + return -EINVAL; + } + + if (unlikely(index->buf >= table->num_hem)) { + dev_err(dev, "table %u exceed hem limt idx %llu, max %llu!\n", + table->type, index->buf, table->num_hem); + return -EINVAL; + } + + return 0; +} + +static void free_mhop_hem(struct udma_dev *udma_dev, + struct udma_hem_table *table, + struct udma_hem_mhop *mhop, + struct udma_hem_index *index) +{ + uint32_t bt_size = mhop->bt_chunk_size; + struct device *dev = udma_dev->dev; + + if (index->inited & HEM_INDEX_BUF) { + udma_free_hem(udma_dev, table->hem[index->buf]); + table->hem[index->buf] = NULL; + } + + if (index->inited & HEM_INDEX_L1) { + dma_free_coherent(dev, bt_size, table->bt_l1[index->l1], + table->bt_l1_dma_addr[index->l1]); + table->bt_l1[index->l1] = NULL; + } + + if (index->inited & HEM_INDEX_L0) { + dma_free_coherent(dev, bt_size, table->bt_l0[index->l0], + table->bt_l0_dma_addr[index->l0]); + table->bt_l0[index->l0] = NULL; + } +} + +static int alloc_mhop_hem(struct udma_dev *udma_dev, + struct udma_hem_table *table, + struct udma_hem_mhop *mhop, + struct udma_hem_index *index) +{ + uint32_t bt_size = mhop->bt_chunk_size; + struct device *dev = udma_dev->dev; + struct udma_hem_iter iter; + uint64_t bt_ba; + uint32_t size; + gfp_t flag; + int ret; + + /* alloc L1 BA's chunk */ + if ((check_whether_bt_num_3(table->type, mhop->hop_num) || + check_whether_bt_num_2(table->type, mhop->hop_num)) && + !table->bt_l0[index->l0]) { + table->bt_l0[index->l0] = dma_alloc_coherent(dev, bt_size, + &table->bt_l0_dma_addr[index->l0], + GFP_KERNEL); + if (!table->bt_l0[index->l0]) { + ret = -ENOMEM; + goto out; + } + index->inited |= HEM_INDEX_L0; + } + + /* alloc L2 BA's chunk */ + if (check_whether_bt_num_3(table->type, mhop->hop_num) && + !table->bt_l1[index->l1]) { + table->bt_l1[index->l1] = dma_alloc_coherent(dev, bt_size, + &table->bt_l1_dma_addr[index->l1], + GFP_KERNEL); + if (!table->bt_l1[index->l1]) { + ret = -ENOMEM; + goto err_alloc_hem; + } + index->inited |= HEM_INDEX_L1; + *(table->bt_l0[index->l0] + mhop->l1_idx) = + table->bt_l1_dma_addr[index->l1]; + } + + /* + * alloc buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC. + * alloc bt space chunk for MTT/CQE. + */ + size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size; + flag = GFP_KERNEL | __GFP_NOWARN; + table->hem[index->buf] = udma_alloc_hem(udma_dev, size >> PAGE_SHIFT, + size, flag); + if (!table->hem[index->buf]) { + ret = -ENOMEM; + goto err_alloc_hem; + } + + index->inited |= HEM_INDEX_BUF; + udma_hem_first(table->hem[index->buf], &iter); + bt_ba = udma_hem_addr(&iter); + if (table->type < HEM_TYPE_MTT) { + if (mhop->hop_num == 2) + *(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba; + else if (mhop->hop_num == 1) + *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; + } else if (mhop->hop_num == 2) { + *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; + } + + return 0; +err_alloc_hem: + free_mhop_hem(udma_dev, table, mhop, index); +out: + return ret; +} + +static int set_mhop_hem(struct udma_dev *udma_dev, + struct udma_hem_table *table, uint64_t obj, + struct udma_hem_mhop *mhop, + struct udma_hem_index *index) +{ + struct device *dev = udma_dev->dev; + int step_idx; + int ret = 0; + + if (index->inited & HEM_INDEX_L0) { + ret = udma_dev->hw->set_hem(udma_dev, table, obj, 0); + if (ret) { + dev_err(dev, "set HEM step 0 failed!\n"); + goto out; + } + } + + if (index->inited & HEM_INDEX_L1) { + ret = udma_dev->hw->set_hem(udma_dev, table, obj, 1); + if (ret) { + dev_err(dev, "set HEM step 1 failed!\n"); + goto out; + } + } + + if (index->inited & HEM_INDEX_BUF) { + if (mhop->hop_num == UDMA_HOP_NUM_0) + step_idx = 0; + else + step_idx = mhop->hop_num; + ret = udma_dev->hw->set_hem(udma_dev, table, obj, step_idx); + if (ret) + dev_err(dev, "set HEM step last failed!\n"); + } +out: + return ret; +} + +static int udma_table_mhop_get(struct udma_dev *udma_dev, + struct udma_hem_table *table, + uint64_t obj) +{ + struct device *dev = udma_dev->dev; + struct udma_hem_index index = {}; + struct udma_hem_mhop mhop = {}; + int ret; + + ret = calc_hem_config(udma_dev, table, obj, &mhop, &index); + if (ret) { + dev_err(dev, "calc hem config failed!\n"); + return ret; + } + + mutex_lock(&table->mutex); + if (table->hem[index.buf]) { + refcount_inc(&table->hem[index.buf]->refcount); + goto out; + } + + ret = alloc_mhop_hem(udma_dev, table, &mhop, &index); + if (ret) { + dev_err(dev, "alloc mhop hem failed!\n"); + goto out; + } + + /* set HEM base address to hardware */ + if (table->type < HEM_TYPE_MTT) { + ret = set_mhop_hem(udma_dev, table, obj, &mhop, &index); + if (ret) { + dev_err(dev, "set HEM address to HW failed!\n"); + goto err_alloc; + } + } + + refcount_set(&table->hem[index.buf]->refcount, 1); + goto out; + +err_alloc: + free_mhop_hem(udma_dev, table, &mhop, &index); +out: + mutex_unlock(&table->mutex); + return ret; +} + +int udma_table_get(struct udma_dev *udma_dev, + struct udma_hem_table *table, uint64_t obj) +{ + struct device *dev = udma_dev->dev; + int ret = 0; + uint64_t i; + + if (udma_check_whether_mhop(udma_dev, table->type)) + return udma_table_mhop_get(udma_dev, table, obj); + + i = obj / (table->table_chunk_size / table->obj_size); + + mutex_lock(&table->mutex); + + if (table->hem[i]) { + refcount_inc(&table->hem[i]->refcount); + goto out; + } + + table->hem[i] = udma_alloc_hem(udma_dev, + table->table_chunk_size >> PAGE_SHIFT, + table->table_chunk_size, + GFP_KERNEL | __GFP_NOWARN); + if (!table->hem[i]) { + ret = -ENOMEM; + goto out; + } + + /* Set HEM base address(128K/page, pa) to Hardware */ + if (udma_dev->hw->set_hem(udma_dev, table, obj, HEM_HOP_STEP_DIRECT)) { + udma_free_hem(udma_dev, table->hem[i]); + table->hem[i] = NULL; + ret = -ENODEV; + dev_err(dev, "set HEM base address to HW failed.\n"); + goto out; + } + + refcount_set(&table->hem[i]->refcount, 1); +out: + mutex_unlock(&table->mutex); + return ret; +} + +static void clear_mhop_hem(struct udma_dev *udma_dev, + struct udma_hem_table *table, uint64_t obj, + struct udma_hem_mhop *mhop, + struct udma_hem_index *index) +{ + struct device *dev = udma_dev->dev; + uint32_t hop_num = mhop->hop_num; + uint32_t chunk_ba_num; + int step_idx; + + index->inited = HEM_INDEX_BUF; + chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; + if (check_whether_bt_num_2(table->type, hop_num)) { + if (udma_check_hem_null(table->hem, index->buf, + chunk_ba_num, table->num_hem)) + index->inited |= HEM_INDEX_L0; + } else if (check_whether_bt_num_3(table->type, hop_num)) { + if (udma_check_hem_null(table->hem, index->buf, + chunk_ba_num, table->num_hem)) { + index->inited |= HEM_INDEX_L1; + if (udma_check_bt_null(table->bt_l1, index->l1, + chunk_ba_num)) + index->inited |= HEM_INDEX_L0; + } + } + + if (table->type < HEM_TYPE_MTT) { + if (hop_num == UDMA_HOP_NUM_0) + step_idx = 0; + else + step_idx = hop_num; + + if (udma_dev->hw->clear_hem(udma_dev, table, obj, step_idx)) + dev_err(dev, "failed to clear hop%u HEM.\n", hop_num); + + if (index->inited & HEM_INDEX_L1) + if (udma_dev->hw->clear_hem(udma_dev, table, obj, 1)) + dev_err(dev, "failed to clear HEM step 1.\n"); + + if (index->inited & HEM_INDEX_L0) + if (udma_dev->hw->clear_hem(udma_dev, table, obj, 0)) + dev_err(dev, "failed to clear HEM step 0.\n"); + } +} + +static void udma_table_mhop_put(struct udma_dev *udma_dev, + struct udma_hem_table *table, uint64_t obj, + int check_refcount) +{ + struct device *dev = udma_dev->dev; + struct udma_hem_index index = {}; + struct udma_hem_mhop mhop = {}; + int ret; + + ret = calc_hem_config(udma_dev, table, obj, &mhop, &index); + if (ret) { + dev_err(dev, "calc hem config failed!\n"); + return; + } + + if (!check_refcount) + mutex_lock(&table->mutex); + else if (!refcount_dec_and_mutex_lock(&table->hem[index.buf]->refcount, + &table->mutex)) + return; + + clear_mhop_hem(udma_dev, table, obj, &mhop, &index); + free_mhop_hem(udma_dev, table, &mhop, &index); + + mutex_unlock(&table->mutex); +} + +void udma_table_put(struct udma_dev *udma_dev, + struct udma_hem_table *table, uint64_t obj) +{ + struct device *dev = udma_dev->dev; + uint64_t i; + + if (udma_check_whether_mhop(udma_dev, table->type)) { + udma_table_mhop_put(udma_dev, table, obj, 1); + return; + } + + i = obj / (table->table_chunk_size / table->obj_size); + + if (!refcount_dec_and_mutex_lock(&table->hem[i]->refcount, + &table->mutex)) + return; + + if (udma_dev->hw->clear_hem(udma_dev, table, obj, HEM_HOP_STEP_DIRECT)) + dev_warn(dev, "failed to clear HEM base address.\n"); + + udma_free_hem(udma_dev, table->hem[i]); + table->hem[i] = NULL; + + mutex_unlock(&table->mutex); +} + +int udma_init_hem_table(struct udma_dev *udma_dev, + struct udma_hem_table *table, uint32_t type, + uint64_t obj_size, uint64_t nobj) +{ + uint64_t obj_per_chunk; + uint64_t num_hem; + + if (!udma_check_whether_mhop(udma_dev, type)) { + table->table_chunk_size = udma_dev->caps.chunk_sz; + obj_per_chunk = table->table_chunk_size / obj_size; + num_hem = DIV_ROUND_UP(nobj, obj_per_chunk); + + table->hem = kcalloc(num_hem, sizeof(*table->hem), + GFP_KERNEL); + if (!table->hem) + return -ENOMEM; + } else { + struct udma_hem_mhop mhop = {}; + uint64_t buf_chunk_size; + uint64_t bt_chunk_size; + uint64_t bt_chunk_num; + uint64_t num_bt_l0; + uint32_t hop_num; + int ret; + + ret = get_hem_table_config(udma_dev, &mhop, type); + if (ret) + return ret; + + buf_chunk_size = mhop.buf_chunk_size; + bt_chunk_size = mhop.bt_chunk_size; + num_bt_l0 = mhop.ba_l0_num; + hop_num = mhop.hop_num; + + obj_per_chunk = buf_chunk_size / obj_size; + num_hem = DIV_ROUND_UP(nobj, obj_per_chunk); + bt_chunk_num = bt_chunk_size / BA_BYTE_LEN; + + if (type >= HEM_TYPE_MTT) + num_bt_l0 = bt_chunk_num; + + table->hem = kcalloc(num_hem, sizeof(*table->hem), + GFP_KERNEL); + if (!table->hem) + goto err_kcalloc_hem_buf; + + if (check_whether_bt_num_3(type, hop_num)) { + uint64_t num_bt_l1; + + num_bt_l1 = DIV_ROUND_UP(num_hem, bt_chunk_num); + table->bt_l1 = kcalloc(num_bt_l1, sizeof(*table->bt_l1), + GFP_KERNEL); + if (!table->bt_l1) + goto err_kcalloc_bt_l1; + + table->bt_l1_dma_addr = kcalloc(num_bt_l1, + sizeof(*table->bt_l1_dma_addr), + GFP_KERNEL); + + if (!table->bt_l1_dma_addr) + goto err_kcalloc_l1_dma; + } + + if (check_whether_bt_num_2(type, hop_num) || + check_whether_bt_num_3(type, hop_num)) { + table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0), + GFP_KERNEL); + if (!table->bt_l0) + goto err_kcalloc_bt_l0; + + table->bt_l0_dma_addr = kcalloc(num_bt_l0, + sizeof(*table->bt_l0_dma_addr), + GFP_KERNEL); + if (!table->bt_l0_dma_addr) + goto err_kcalloc_l0_dma; + } + } + + table->type = type; + table->num_hem = num_hem; + table->obj_size = obj_size; + mutex_init(&table->mutex); + + return 0; + +err_kcalloc_l0_dma: + kfree(table->bt_l0); + table->bt_l0 = NULL; + +err_kcalloc_bt_l0: + kfree(table->bt_l1_dma_addr); + table->bt_l1_dma_addr = NULL; + +err_kcalloc_l1_dma: + kfree(table->bt_l1); + table->bt_l1 = NULL; + +err_kcalloc_bt_l1: + kfree(table->hem); + table->hem = NULL; + +err_kcalloc_hem_buf: + return -ENOMEM; +} + +static void udma_cleanup_mhop_hem_table(struct udma_dev *udma_dev, + struct udma_hem_table *table) +{ + struct udma_hem_mhop mhop; + uint32_t buf_chunk_size; + uint64_t obj; + uint32_t i; + + if (udma_calc_hem_mhop(udma_dev, table, NULL, &mhop)) + return; + buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size : + mhop.bt_chunk_size; + + for (i = 0; i < table->num_hem; ++i) { + obj = i * buf_chunk_size / table->obj_size; + if (table->hem[i]) + udma_table_mhop_put(udma_dev, table, obj, 0); + } + + kfree(table->hem); + table->hem = NULL; + kfree(table->bt_l1); + table->bt_l1 = NULL; + kfree(table->bt_l1_dma_addr); + table->bt_l1_dma_addr = NULL; + kfree(table->bt_l0); + table->bt_l0 = NULL; + kfree(table->bt_l0_dma_addr); + table->bt_l0_dma_addr = NULL; +} + +void udma_cleanup_hem_table(struct udma_dev *udma_dev, + struct udma_hem_table *table) +{ + struct device *dev = udma_dev->dev; + uint64_t i; + + if (udma_check_whether_mhop(udma_dev, table->type)) { + udma_cleanup_mhop_hem_table(udma_dev, table); + return; + } + + for (i = 0; i < table->num_hem; ++i) + if (table->hem[i]) { + if (udma_dev->hw->clear_hem(udma_dev, table, + i * table->table_chunk_size + / table->obj_size, 0)) + dev_err(dev, "Clear HEM base address failed.\n"); + + udma_free_hem(udma_dev, table->hem[i]); + } + + kfree(table->hem); +} + +struct udma_buf *udma_buf_alloc(struct udma_dev *udma_dev, uint32_t size, + uint32_t page_shift, uint32_t flags) +{ + uint32_t trunk_size, page_size, alloced_size; + struct udma_buf_list *trunks; + struct udma_buf *buf; + uint32_t ntrunk, i; + gfp_t gfp_flags; + + /* The minimum shift of the page accessed by hw is UDMA_PAGE_SHIFT */ + if (WARN_ON(page_shift < UDMA_HW_PAGE_SHIFT)) + return ERR_PTR(-EINVAL); + + gfp_flags = (flags & UDMA_BUF_NOSLEEP) ? GFP_ATOMIC : GFP_KERNEL; + buf = kzalloc(sizeof(*buf), gfp_flags); + if (!buf) + return ERR_PTR(-ENOMEM); + + buf->page_shift = page_shift; + page_size = 1 << buf->page_shift; + + /* Calc the trunk size and num by required size and page_shift */ + if (flags & UDMA_BUF_DIRECT) { + buf->trunk_shift = order_base_2(ALIGN(size, PAGE_SIZE)); + ntrunk = 1; + } else { + buf->trunk_shift = order_base_2(ALIGN(page_size, PAGE_SIZE)); + ntrunk = DIV_ROUND_UP(size, 1 << buf->trunk_shift); + } + + trunks = kcalloc(ntrunk, sizeof(*trunks), gfp_flags); + if (!trunks) { + kfree(buf); + return ERR_PTR(-ENOMEM); + } + + trunk_size = 1 << buf->trunk_shift; + alloced_size = 0; + for (i = 0; i < ntrunk; i++) { + trunks[i].buf = dma_alloc_coherent(udma_dev->dev, trunk_size, + &trunks[i].map, gfp_flags); + if (!trunks[i].buf) + break; + + alloced_size += trunk_size; + } + + buf->ntrunks = i; + + /* In nofail mode, it's only failed when the alloced size is 0 */ + if ((flags & UDMA_BUF_NOFAIL) ? i == 0 : i != ntrunk) { + for (i = 0; i < buf->ntrunks; i++) + dma_free_coherent(udma_dev->dev, trunk_size, + trunks[i].buf, trunks[i].map); + + kfree(trunks); + kfree(buf); + return ERR_PTR(-ENOMEM); + } + + buf->npages = DIV_ROUND_UP(alloced_size, page_size); + buf->trunk_list = trunks; + + return buf; +} +void udma_buf_free(struct udma_dev *udma_dev, struct udma_buf *buf) +{ + struct udma_buf_list *trunks; + uint32_t i; + + if (!buf) + return; + + trunks = buf->trunk_list; + if (trunks) { + buf->trunk_list = NULL; + for (i = 0; i < buf->ntrunks; i++) + dma_free_coherent(udma_dev->dev, 1 << buf->trunk_shift, + trunks[i].buf, trunks[i].map); + + kfree(trunks); + } + + kfree(buf); +} diff --git a/drivers/ub/hw/hns3/hns3_udma_hem.h b/drivers/ub/hw/hns3/hns3_udma_hem.h index 1bef0304fbec..dbf8b594ede4 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hem.h +++ b/drivers/ub/hw/hns3/hns3_udma_hem.h @@ -16,6 +16,20 @@ #ifndef _UDMA_HEM_H #define _UDMA_HEM_H +#include +#include +#define HEM_HOP_STEP_DIRECT 0xff +#define HEM_INDEX_BUF BIT(0) +#define HEM_INDEX_L0 BIT(1) +#define HEM_INDEX_L1 BIT(2) + +struct udma_hem_index { + uint64_t buf; + uint64_t l0; + uint64_t l1; + uint32_t inited; /* indicate which index is available */ +}; + enum { /* UDMA MAP HEM(Hardware Entry Memory) */ HEM_TYPE_QPC = 0, @@ -24,12 +38,117 @@ enum { HEM_TYPE_SRQC, /* SCC CONTEXT */ HEM_TYPE_SCCC, + HEM_TYPE_QPC_TIMER, + HEM_TYPE_CQC_TIMER, + HEM_TYPE_GMV, /* UDMA UNMAP HEM */ HEM_TYPE_MTT, HEM_TYPE_CQE, HEM_TYPE_SRQWQE, HEM_TYPE_INDEX, + HEM_TYPE_IRRL, + HEM_TYPE_TRRL, +}; + +#define UDMA_HEM_CHUNK_LEN \ + ((256 - sizeof(struct list_head) - 2 * sizeof(int)) / \ + (sizeof(struct scatterlist) + sizeof(void *))) + +#define check_whether_bt_num_3(type, hop_num) \ + (type < HEM_TYPE_MTT && (hop_num) == 2) + +#define check_whether_bt_num_2(type, hop_num) \ + ((type < HEM_TYPE_MTT && (hop_num) == 1) || \ + (type >= HEM_TYPE_MTT && (hop_num) == 2)) + +#define check_whether_bt_num_1(type, hop_num) \ + (((hop_num) == UDMA_HOP_NUM_0) || \ + (type >= HEM_TYPE_MTT && (hop_num) == 1)) + +#define check_whether_last_step(hop_num, step_idx) \ + (((step_idx) == 0 && (hop_num) == UDMA_HOP_NUM_0) || \ + ((step_idx) == 1 && (hop_num) == 1) || \ + ((step_idx) == 2 && (hop_num) == 2)) + +struct udma_hem_chunk { + struct list_head list; + int npages; + int nsg; + struct scatterlist mem[UDMA_HEM_CHUNK_LEN]; + void *buf[UDMA_HEM_CHUNK_LEN]; +}; + +struct udma_hem { + struct list_head chunk_list; + refcount_t refcount; +}; + +struct udma_hem_iter { + struct udma_hem *hem; + struct udma_hem_chunk *chunk; + int page_idx; }; -#endif +struct udma_hem_mhop { + uint32_t hop_num; + uint32_t buf_chunk_size; + uint32_t bt_chunk_size; + uint32_t ba_l0_num; + uint32_t l0_idx; /* level 0 base address table index */ + uint32_t l1_idx; /* level 1 base address table index */ + uint32_t l2_idx; /* level 2 base address table index */ +}; + +struct udma_buf *udma_buf_alloc(struct udma_dev *udma_dev, uint32_t size, + uint32_t page_shift, uint32_t flags); +void udma_buf_free(struct udma_dev *udma_dev, struct udma_buf *buf); +int udma_table_get(struct udma_dev *udma_dev, + struct udma_hem_table *table, uint64_t obj); +void udma_table_put(struct udma_dev *udma_dev, + struct udma_hem_table *table, uint64_t obj); +int udma_init_hem_table(struct udma_dev *udma_dev, + struct udma_hem_table *table, uint32_t type, + uint64_t obj_size, uint64_t nobj); +void udma_cleanup_hem_table(struct udma_dev *udma_dev, + struct udma_hem_table *table); +int udma_calc_hem_mhop(struct udma_dev *udma_dev, + struct udma_hem_table *table, uint64_t *obj, + struct udma_hem_mhop *mhop); +bool udma_check_whether_mhop(struct udma_dev *udma_dev, uint32_t type); + +static inline void udma_hem_first(struct udma_hem *hem, + struct udma_hem_iter *iter) +{ + iter->hem = hem; + iter->chunk = list_empty(&hem->chunk_list) ? NULL : + list_entry(hem->chunk_list.next, struct udma_hem_chunk, + list); + iter->page_idx = 0; +} + +static inline int udma_hem_last(struct udma_hem_iter *iter) +{ + return !iter->chunk; +} + +static inline void udma_hem_next(struct udma_hem_iter *iter) +{ + if (++iter->page_idx >= iter->chunk->nsg) { + if (iter->chunk->list.next == &iter->hem->chunk_list) { + iter->chunk = NULL; + return; + } + + iter->chunk = list_entry(iter->chunk->list.next, + struct udma_hem_chunk, list); + iter->page_idx = 0; + } +} + +static inline dma_addr_t udma_hem_addr(struct udma_hem_iter *iter) +{ + return sg_dma_address(&iter->chunk->mem[iter->page_idx]); +} + +#endif /* _UDMA_HEM_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.c b/drivers/ub/hw/hns3/hns3_udma_hw.c index efcb933a79b9..95418c09a8c0 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hw.c +++ b/drivers/ub/hw/hns3/hns3_udma_hw.c @@ -18,6 +18,7 @@ #include #include "hnae3.h" #include "hns3_udma_cmd.h" +#include "hns3_udma_hem.h" static const struct pci_device_id udma_hw_pci_tbl[] = { { PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_UDMA_OVER_UBL), @@ -339,6 +340,500 @@ static int udma_query_caps(struct udma_dev *udma_dev) return 0; } +static int load_func_res_caps(struct udma_dev *udma_dev) +{ + struct udma_cmq_desc desc[2]; + struct udma_cmq_req *r_a = (struct udma_cmq_req *)desc[0].data; + struct udma_cmq_req *r_b = (struct udma_cmq_req *)desc[1].data; + struct udma_caps *caps = &udma_dev->caps; + enum udma_opcode_type opcode; + uint32_t func_num; + int ret; + + opcode = UDMA_OPC_QUERY_PF_RES; + func_num = udma_dev->func_num; + + udma_cmq_setup_basic_desc(&desc[0], opcode, true); + desc[0].flag |= cpu_to_le16(UDMA_CMD_FLAG_NEXT); + udma_cmq_setup_basic_desc(&desc[1], opcode, true); + + ret = udma_cmq_send(udma_dev, desc, 2); + if (ret) + return ret; + + caps->qpc_bt_num = udma_reg_read(r_a, FUNC_RES_A_QPC_BT_NUM) / func_num; + caps->srqc_bt_num = + udma_reg_read(r_a, FUNC_RES_A_SRQC_BT_NUM) / func_num; + caps->cqc_bt_num = udma_reg_read(r_a, FUNC_RES_A_CQC_BT_NUM) / func_num; + caps->mpt_bt_num = udma_reg_read(r_a, FUNC_RES_A_MPT_BT_NUM) / func_num; + caps->eqc_bt_num = udma_reg_read(r_a, FUNC_RES_A_EQC_BT_NUM) / func_num; + caps->smac_bt_num = udma_reg_read(r_b, FUNC_RES_B_SMAC_NUM) / func_num; + caps->sgid_bt_num = udma_reg_read(r_b, FUNC_RES_B_SGID_NUM) / func_num; + caps->sccc_bt_num = + udma_reg_read(r_b, FUNC_RES_B_SCCC_BT_NUM) / func_num; + + + caps->sl_num = + udma_reg_read(r_b, FUNC_RES_B_QID_NUM) / func_num; + caps->gmv_bt_num = + udma_reg_read(r_b, FUNC_RES_B_GMV_BT_NUM) / func_num; + + return 0; +} + +static void setup_default_ext_caps(struct udma_dev *udma_dev) +{ + struct udma_caps *caps = &udma_dev->caps; + + caps->num_pi_qps = caps->num_qps; + caps->llm_ba_idx = 0; + caps->llm_ba_num = UDMA_EXT_LLM_MAX_DEPTH; + +} + +static int load_ext_cfg_caps(struct udma_dev *udma_dev) +{ + struct udma_cmq_desc desc; + struct udma_cmq_req *req = (struct udma_cmq_req *)desc.data; + struct udma_caps *caps = &udma_dev->caps; + uint32_t func_num, qp_num; + int ret; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_EXT_CFG, true); + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) + return ret; + + func_num = max_t(uint32_t, 1, udma_dev->func_num); + qp_num = udma_reg_read(req, EXT_CFG_QP_PI_NUM) / func_num; + caps->num_pi_qps = round_down(qp_num, UDMA_QP_BANK_NUM); + + qp_num = udma_reg_read(req, EXT_CFG_QP_NUM) / func_num; + caps->num_qps = round_down(qp_num, UDMA_QP_BANK_NUM); + caps->num_qps_shift = ilog2(caps->num_qps); + + /* The extend doorbell memory on the PF is shared by all its VFs. */ + caps->llm_ba_idx = udma_reg_read(req, EXT_CFG_LLM_IDX); + caps->llm_ba_num = udma_reg_read(req, EXT_CFG_LLM_NUM); + + return 0; +} + +static int query_func_resource_caps(struct udma_dev *udma_dev) +{ + struct device *dev = udma_dev->dev; + int ret; + + ret = load_func_res_caps(udma_dev); + if (ret) { + dev_err(dev, "failed to load res caps, ret = %d (pf).\n", + ret); + return ret; + } + + setup_default_ext_caps(udma_dev); + + ret = load_ext_cfg_caps(udma_dev); + if (ret) + dev_err(dev, "failed to load ext cfg, ret = %d (pf).\n", + ret); + return ret; +} + +static int load_pf_timer_res_caps(struct udma_dev *udma_dev) +{ + struct udma_cmq_desc desc; + struct udma_cmq_req *req = (struct udma_cmq_req *)desc.data; + struct udma_caps *caps = &udma_dev->caps; + int ret; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_QUERY_PF_TIMER_RES, + true); + + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) + return ret; + + caps->qpc_timer_bt_num = udma_reg_read(req, PF_TIMER_RES_QPC_ITEM_NUM); + caps->cqc_timer_bt_num = udma_reg_read(req, PF_TIMER_RES_CQC_ITEM_NUM); + + return 0; +} + +static int query_func_oor_caps(struct udma_dev *udma_dev) +{ + struct udma_caps *caps = &udma_dev->caps; + struct udma_query_oor_cmq *resp; + struct udma_cmq_desc desc; + int ret = 0; + + udma_cmq_setup_basic_desc(&desc, UDMA_QUERY_OOR_CAPS, true); + + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) { + dev_err(udma_dev->dev, + "failed to func oor caps, ret = %d.\n", ret); + return ret; + } + + resp = (struct udma_query_oor_cmq *)desc.data; + + caps->oor_en = resp->oor_en; + caps->reorder_cq_buffer_en = resp->reorder_cq_buffer_en; + caps->reorder_cap = resp->reorder_cap; + caps->reorder_cq_shift = resp->reorder_cq_shift; + caps->onflight_size = resp->on_flight_size; + caps->dynamic_ack_timeout = resp->dynamic_ack_timeout; + + return ret; +} + +static int udma_query_pf_resource(struct udma_dev *udma_dev) +{ + struct device *dev = udma_dev->dev; + int ret; + + ret = query_func_resource_caps(udma_dev); + if (ret) + return ret; + + ret = load_pf_timer_res_caps(udma_dev); + if (ret) { + dev_err(dev, "failed to load pf timer resource, ret = %d.\n", + ret); + return ret; + } + + if (udma_dev->caps.flags & UDMA_CAP_FLAG_OOR) + ret = query_func_oor_caps(udma_dev); + + return ret; +} + +static void calc_pg_sz(uint32_t obj_num, uint32_t obj_size, uint32_t hop_num, + uint32_t ctx_bt_num, uint32_t *buf_page_size, + uint32_t *bt_page_size, uint32_t hem_type) +{ + uint64_t buf_chunk_size = PAGE_SIZE; + uint64_t bt_chunk_size = PAGE_SIZE; + uint64_t obj_per_chunk_default; + uint64_t obj_per_chunk; + + obj_per_chunk_default = buf_chunk_size / obj_size; + *buf_page_size = 0; + *bt_page_size = 0; + + switch (hop_num) { + case 3: + obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) * + (bt_chunk_size / BA_BYTE_LEN) * + (bt_chunk_size / BA_BYTE_LEN) * + obj_per_chunk_default; + break; + case 2: + obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) * + (bt_chunk_size / BA_BYTE_LEN) * + obj_per_chunk_default; + break; + case 1: + obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) * + obj_per_chunk_default; + break; + case UDMA_HOP_NUM_0: + obj_per_chunk = ctx_bt_num * obj_per_chunk_default; + break; + default: + pr_err("table %u not support hop_num = %u!\n", hem_type, + hop_num); + return; + } + + if (hem_type >= HEM_TYPE_MTT) + *bt_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk)); + else + *buf_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk)); +} + +static void set_hem_page_size(struct udma_dev *udma_dev) +{ + struct udma_caps *caps = &udma_dev->caps; + + /* EQ */ + caps->eqe_ba_pg_sz = 0; + caps->eqe_buf_pg_sz = 0; + + /* Link Table */ + caps->llm_buf_pg_sz = 0; + + /* MR */ + caps->mpt_ba_pg_sz = 0; + caps->mpt_buf_pg_sz = 0; + caps->pbl_ba_pg_sz = UDMA_BA_PG_SZ_SUPPORTED_16K; + caps->pbl_buf_pg_sz = 0; + calc_pg_sz(caps->num_mtpts, caps->mtpt_entry_sz, caps->mpt_hop_num, + caps->mpt_bt_num, &caps->mpt_buf_pg_sz, &caps->mpt_ba_pg_sz, + HEM_TYPE_MTPT); + + /* QP */ + caps->qpc_ba_pg_sz = 0; + caps->qpc_buf_pg_sz = 0; + caps->qpc_timer_ba_pg_sz = 0; + caps->qpc_timer_buf_pg_sz = 0; + caps->sccc_ba_pg_sz = 0; + caps->sccc_buf_pg_sz = 0; + caps->mtt_ba_pg_sz = 0; + caps->mtt_buf_pg_sz = 0; + calc_pg_sz(caps->num_qps, caps->qpc_sz, caps->qpc_hop_num, + caps->qpc_bt_num, &caps->qpc_buf_pg_sz, &caps->qpc_ba_pg_sz, + HEM_TYPE_QPC); + + if (caps->flags & UDMA_CAP_FLAG_QP_FLOW_CTRL) + calc_pg_sz(caps->num_qps, caps->scc_ctx_sz, caps->sccc_hop_num, + caps->sccc_bt_num, &caps->sccc_buf_pg_sz, + &caps->sccc_ba_pg_sz, HEM_TYPE_SCCC); + + /* CQ */ + caps->cqc_ba_pg_sz = 0; + caps->cqc_buf_pg_sz = 0; + caps->cqc_timer_ba_pg_sz = 0; + caps->cqc_timer_buf_pg_sz = 0; + caps->cqe_ba_pg_sz = UDMA_BA_PG_SZ_SUPPORTED_256K; + caps->cqe_buf_pg_sz = 0; + calc_pg_sz(caps->num_cqs, caps->cqc_entry_sz, caps->cqc_hop_num, + caps->cqc_bt_num, &caps->cqc_buf_pg_sz, &caps->cqc_ba_pg_sz, + HEM_TYPE_CQC); + calc_pg_sz(caps->max_cqes, caps->cqe_sz, caps->cqe_hop_num, + 1, &caps->cqe_buf_pg_sz, &caps->cqe_ba_pg_sz, HEM_TYPE_CQE); + + /* SRQ */ + if (caps->flags & UDMA_CAP_FLAG_SRQ) { + caps->srqc_ba_pg_sz = 0; + caps->srqc_buf_pg_sz = 0; + caps->srqwqe_ba_pg_sz = 0; + caps->srqwqe_buf_pg_sz = 0; + caps->idx_ba_pg_sz = 0; + caps->idx_buf_pg_sz = 0; + calc_pg_sz(caps->num_srqs, caps->srqc_entry_sz, + caps->srqc_hop_num, caps->srqc_bt_num, + &caps->srqc_buf_pg_sz, &caps->srqc_ba_pg_sz, + HEM_TYPE_SRQC); + calc_pg_sz(caps->num_srqwqe_segs, caps->mtt_entry_sz, + caps->srqwqe_hop_num, 1, &caps->srqwqe_buf_pg_sz, + &caps->srqwqe_ba_pg_sz, HEM_TYPE_SRQWQE); + calc_pg_sz(caps->num_idx_segs, caps->idx_entry_sz, + caps->idx_hop_num, 1, &caps->idx_buf_pg_sz, + &caps->idx_ba_pg_sz, HEM_TYPE_INDEX); + } + + /* GMV */ + caps->gmv_ba_pg_sz = 0; + caps->gmv_buf_pg_sz = 0; +} + +/* Apply all loaded caps before setting to hardware */ +static void apply_func_caps(struct udma_dev *udma_dev) +{ + struct udma_caps *caps = &udma_dev->caps; + struct udma_priv *priv = (struct udma_priv *)udma_dev->priv; + + caps->qpc_timer_entry_sz = UDMA_QPC_TIMER_ENTRY_SZ; + caps->cqc_timer_entry_sz = UDMA_CQC_TIMER_ENTRY_SZ; + caps->mtt_entry_sz = UDMA_MTT_ENTRY_SZ; + caps->eqe_hop_num = UDMA_EQE_HOP_NUM; + caps->pbl_hop_num = UDMA_PBL_HOP_NUM; + caps->qpc_timer_hop_num = UDMA_HOP_NUM_0; + caps->cqc_timer_hop_num = UDMA_HOP_NUM_0; + caps->ceqe_size = UDMA_EQE_SIZE; + caps->aeqe_size = UDMA_EQE_SIZE; + caps->num_mtt_segs = UDMA_MAX_MTT_SEGS; + caps->num_srqwqe_segs = UDMA_MAX_SRQWQE_SEGS; + caps->num_idx_segs = UDMA_MAX_IDX_SEGS; + + /* num_vector = comp_vector + aeq_vector + abn_vector */ + if (!caps->num_comp_vectors) { + caps->num_comp_vectors = + min_t(uint32_t, caps->eqc_bt_num - 1, + (uint32_t)priv->handle->udmainfo.num_vectors - + UDMA_FUNC_IRQ_RSV); + } + caps->qpc_sz = UDMA_QPC_SZ; + caps->cqe_sz = UDMA_CQE_SZ; + caps->scc_ctx_sz = UDMA_SCCC_SZ; + + /* The following caps are not in ncl config */ + caps->gmv_entry_sz = UDMA_GMV_ENTRY_SZ; + + caps->gmv_hop_num = UDMA_HOP_NUM_0; + caps->gid_table_len[0] = caps->gmv_bt_num * + (UDMA_PAGE_SIZE / caps->gmv_entry_sz); + + caps->gmv_entry_num = caps->gmv_bt_num * (PAGE_SIZE / + caps->gmv_entry_sz); + + set_hem_page_size(udma_dev); +} + +static int config_vf_ext_resource(struct udma_dev *udma_dev, uint32_t vf_id) +{ + struct udma_cmq_desc desc; + struct udma_cmq_req *req = (struct udma_cmq_req *)desc.data; + struct udma_caps *caps = &udma_dev->caps; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_EXT_CFG, false); + + udma_reg_write(req, EXT_CFG_VF_ID, vf_id); + + udma_reg_write(req, EXT_CFG_QP_PI_NUM, caps->num_pi_qps); + udma_reg_write(req, EXT_CFG_QP_PI_INDEX, vf_id * caps->num_pi_qps); + udma_reg_write(req, EXT_CFG_QP_NUM, caps->num_qps); + udma_reg_write(req, EXT_CFG_QP_INDEX, vf_id * caps->num_qps); + + return udma_cmq_send(udma_dev, &desc, 1); +} + +static int config_vf_hem_resource(struct udma_dev *udma_dev, int vf_id) +{ + struct udma_cmq_desc desc[2]; + struct udma_cmq_req *r_a = (struct udma_cmq_req *)desc[0].data; + struct udma_cmq_req *r_b = (struct udma_cmq_req *)desc[1].data; + struct udma_caps *caps = &udma_dev->caps; + + udma_cmq_setup_basic_desc(&desc[0], UDMA_OPC_ALLOC_VF_RES, false); + desc[0].flag |= cpu_to_le16(UDMA_CMD_FLAG_NEXT); + udma_cmq_setup_basic_desc(&desc[1], UDMA_OPC_ALLOC_VF_RES, false); + + udma_reg_write(r_a, FUNC_RES_A_VF_ID, vf_id); + + udma_reg_write(r_a, FUNC_RES_A_QPC_BT_NUM, caps->qpc_bt_num); + udma_reg_write(r_a, FUNC_RES_A_QPC_BT_INDEX, vf_id * caps->qpc_bt_num); + udma_reg_write(r_a, FUNC_RES_A_SRQC_BT_NUM, caps->srqc_bt_num); + udma_reg_write(r_a, FUNC_RES_A_SRQC_BT_INDEX, vf_id * caps->srqc_bt_num); + udma_reg_write(r_a, FUNC_RES_A_CQC_BT_NUM, caps->cqc_bt_num); + udma_reg_write(r_a, FUNC_RES_A_CQC_BT_INDEX, vf_id * caps->cqc_bt_num); + udma_reg_write(r_a, FUNC_RES_A_MPT_BT_NUM, caps->mpt_bt_num); + udma_reg_write(r_a, FUNC_RES_A_MPT_BT_INDEX, vf_id * caps->mpt_bt_num); + udma_reg_write(r_a, FUNC_RES_A_EQC_BT_NUM, caps->eqc_bt_num); + udma_reg_write(r_a, FUNC_RES_A_EQC_BT_INDEX, vf_id * caps->eqc_bt_num); + udma_reg_write(r_b, FUNC_RES_V_QID_NUM, caps->sl_num); + udma_reg_write(r_b, FUNC_RES_B_QID_INDEX, vf_id * caps->sl_num); + udma_reg_write(r_b, FUNC_RES_B_SCCC_BT_NUM, caps->sccc_bt_num); + udma_reg_write(r_b, FUNC_RES_B_SCCC_BT_INDEX, vf_id * caps->sccc_bt_num); + udma_reg_write(r_b, FUNC_RES_V_GMV_BT_NUM, caps->gmv_bt_num); + udma_reg_write(r_b, FUNC_RES_B_GMV_BT_INDEX, + vf_id * caps->gmv_bt_num); + + return udma_cmq_send(udma_dev, desc, 2); +} + +static int udma_alloc_vf_resource(struct udma_dev *udma_dev) +{ + uint32_t vf_id; + int ret; + + for (vf_id = 0; vf_id < udma_dev->func_num; vf_id++) { + ret = config_vf_hem_resource(udma_dev, vf_id); + if (ret) { + dev_err(udma_dev->dev, + "failed to config vf-%u hem res, ret = %d.\n", + vf_id, ret); + return ret; + } + + ret = config_vf_ext_resource(udma_dev, vf_id); + if (ret) { + dev_err(udma_dev->dev, + "failed to config vf-%u ext res, ret = %d.\n", + vf_id, ret); + return ret; + } + } + + return 0; +} + +static int udma_set_bt(struct udma_dev *udma_dev) +{ + struct udma_cmq_desc desc; + struct udma_cmq_req *req = (struct udma_cmq_req *)desc.data; + struct udma_caps *caps = &udma_dev->caps; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_CFG_BT_ATTR, false); + + udma_reg_write(req, CFG_BT_QPC_BA_PGSZ, + caps->qpc_ba_pg_sz + PG_SHIFT_OFFSET); + udma_reg_write(req, CFG_BT_QPC_BUF_PGSZ, + caps->qpc_buf_pg_sz + PG_SHIFT_OFFSET); + udma_reg_write(req, CFG_BT_QPC_HOPNUM, + to_udma_hem_hopnum(caps->qpc_hop_num, caps->num_qps)); + + udma_reg_write(req, CFG_BT_SRQC_BA_PGSZ, + caps->srqc_ba_pg_sz + PG_SHIFT_OFFSET); + udma_reg_write(req, CFG_BT_SRQC_BUF_PGSZ, + caps->srqc_buf_pg_sz + PG_SHIFT_OFFSET); + udma_reg_write(req, CFG_BT_SRQC_HOPNUM, + to_udma_hem_hopnum(caps->srqc_hop_num, caps->num_srqs)); + + udma_reg_write(req, CFG_BT_CQC_BA_PGSZ, + caps->cqc_ba_pg_sz + PG_SHIFT_OFFSET); + udma_reg_write(req, CFG_BT_CQC_BUF_PGSZ, + caps->cqc_buf_pg_sz + PG_SHIFT_OFFSET); + udma_reg_write(req, CFG_BT_CQC_HOPNUM, + to_udma_hem_hopnum(caps->cqc_hop_num, caps->num_cqs)); + + udma_reg_write(req, CFG_BT_MPT_BA_PGSZ, + caps->mpt_ba_pg_sz + PG_SHIFT_OFFSET); + udma_reg_write(req, CFG_BT_MPT_BUF_PGSZ, + caps->mpt_buf_pg_sz + PG_SHIFT_OFFSET); + udma_reg_write(req, CFG_BT_MPT_HOPNUM, + to_udma_hem_hopnum(caps->mpt_hop_num, caps->num_mtpts)); + + udma_reg_write(req, CFG_BT_SCCC_BA_PGSZ, + caps->sccc_ba_pg_sz + PG_SHIFT_OFFSET); + udma_reg_write(req, CFG_BT_SCCC_BUF_PGSZ, + caps->sccc_buf_pg_sz + PG_SHIFT_OFFSET); + udma_reg_write(req, CFG_BT_SCCC_HOPNUM, + to_udma_hem_hopnum(caps->sccc_hop_num, caps->num_qps)); + + return udma_cmq_send(udma_dev, &desc, 1); +} + +static int config_hem_entry_size(struct udma_dev *udma_dev, uint32_t type, + uint32_t val) +{ + struct udma_cmq_desc desc; + struct udma_cmq_req *req = (struct udma_cmq_req *)desc.data; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_CFG_ENTRY_SIZE, + false); + + udma_reg_write(req, CFG_HEM_ENTRY_SIZE_TYPE, type); + udma_reg_write(req, CFG_HEM_ENTRY_SIZE_VALUE, val); + + return udma_cmq_send(udma_dev, &desc, 1); +} + +static int udma_config_entry_size(struct udma_dev *udma_dev) +{ + struct udma_caps *caps = &udma_dev->caps; + int ret; + + ret = config_hem_entry_size(udma_dev, UDMA_CFG_QPC_SIZE, + caps->qpc_sz); + if (ret) { + dev_err(udma_dev->dev, + "failed to cfg qpc sz, ret = %d.\n", ret); + return ret; + } + + ret = config_hem_entry_size(udma_dev, UDMA_CFG_SCCC_SIZE, + caps->scc_ctx_sz); + if (ret) + dev_err(udma_dev->dev, + "failed to cfg sccc sz, ret = %d.\n", ret); + + return ret; +} + static int udma_pf_profile(struct udma_dev *udma_dev) { struct device *dev = udma_dev->dev; @@ -368,7 +863,27 @@ static int udma_pf_profile(struct udma_dev *udma_dev) return ret; } - return 0; + ret = udma_query_pf_resource(udma_dev); + if (ret) { + dev_err(dev, "failed to query pf resource, ret = %d.\n", ret); + return ret; + } + + apply_func_caps(udma_dev); + ret = udma_alloc_vf_resource(udma_dev); + if (ret) { + dev_err(dev, "failed to alloc vf resource, ret = %d.\n", ret); + return ret; + } + + ret = udma_set_bt(udma_dev); + if (ret) { + dev_err(dev, "failed to config BA table, ret = %d.\n", ret); + return ret; + } + + /* Configure the size of QPC, SCCC, etc. */ + return udma_config_entry_size(udma_dev); } static int udma_profile(struct udma_dev *udma_dev) @@ -478,14 +993,444 @@ static void udma_cmq_exit(struct udma_dev *udma_dev) udma_free_cmq_desc(udma_dev, &priv->cmq.csq); } + +static void config_llm_table(struct udma_buf *data_buf, void *cfg_buf) +{ + uint64_t *entry = (uint64_t *)cfg_buf; + uint32_t i, next_ptr, page_num; + dma_addr_t addr; + uint64_t val; + + page_num = data_buf->npages; + for (i = 0; i < page_num; i++) { + addr = udma_buf_page(data_buf, i); + if (i == (page_num - 1)) + next_ptr = 0; + else + next_ptr = i + 1; + + val = UDMA_EXT_LLM_ENTRY(addr, (uint64_t)next_ptr); + entry[i] = cpu_to_le64(val); + } +} + +static int set_llm_cfg_to_hw(struct udma_dev *udma_dev, + struct udma_link_table *table) +{ + struct udma_cmq_desc desc[2]; + struct udma_cmq_req *r_a = (struct udma_cmq_req *)desc[0].data; + struct udma_cmq_req *r_b = (struct udma_cmq_req *)desc[1].data; + struct udma_buf *buf = table->buf; + enum udma_opcode_type opcode; + dma_addr_t addr; + + opcode = UDMA_OPC_CFG_EXT_LLM; + udma_cmq_setup_basic_desc(&desc[0], opcode, false); + desc[0].flag |= cpu_to_le16(UDMA_CMD_FLAG_NEXT); + udma_cmq_setup_basic_desc(&desc[1], opcode, false); + + udma_reg_write(r_a, CFG_LLM_A_BA_L, lower_32_bits(table->table.map)); + udma_reg_write(r_a, CFG_LLM_A_BA_H, upper_32_bits(table->table.map)); + udma_reg_write(r_a, CFG_LLM_A_DEPTH, buf->npages); + udma_reg_write(r_a, CFG_LLM_A_PG_SZ, + to_hr_hw_page_shift(buf->page_shift)); + udma_reg_enable(r_a, CFG_LLM_A_INIT_EN); + + addr = to_hr_hw_page_addr(udma_buf_page(buf, 0)); + udma_reg_write(r_a, CFG_LLM_A_HEAD_BA_L, lower_32_bits(addr)); + udma_reg_write(r_a, CFG_LLM_A_HEAD_BA_H, upper_32_bits(addr)); + udma_reg_write(r_a, CFG_LLM_A_HEAD_NXT_PTR, 1); + udma_reg_write(r_a, CFG_LLM_A_HEAD_PTR, 0); + + addr = to_hr_hw_page_addr(udma_buf_page(buf, buf->npages - 1)); + udma_reg_write(r_b, CFG_LLM_B_TAIL_BA_L, lower_32_bits(addr)); + udma_reg_write(r_b, CFG_LLM_B_TAIL_BA_H, upper_32_bits(addr)); + udma_reg_write(r_b, CFG_LLM_B_TAIL_PTR, buf->npages - 1); + + return udma_cmq_send(udma_dev, desc, 2); +} + +static struct udma_link_table * +alloc_link_table_buf(struct udma_dev *udma_dev) +{ + struct udma_priv *priv = (struct udma_priv *)udma_dev->priv; + uint32_t pg_shift, size, min_size; + struct udma_link_table *link_tbl; + + link_tbl = &priv->ext_llm; + pg_shift = udma_dev->caps.llm_buf_pg_sz + PAGE_SHIFT; + size = udma_dev->caps.num_qps * UDMA_EXT_LLM_ENTRY_SZ; + min_size = UDMA_EXT_LLM_MIN_PAGES(udma_dev->caps.sl_num) << pg_shift; + + /* Alloc data table */ + size = max(size, min_size); + link_tbl->buf = udma_buf_alloc(udma_dev, size, pg_shift, 0); + if (IS_ERR(link_tbl->buf)) + return ERR_PTR(-ENOMEM); + + /* Alloc config table */ + size = link_tbl->buf->npages * sizeof(uint64_t); + link_tbl->table.buf = dma_alloc_coherent(udma_dev->dev, size, + &link_tbl->table.map, + GFP_KERNEL); + if (!link_tbl->table.buf) { + udma_buf_free(udma_dev, link_tbl->buf); + return ERR_PTR(-ENOMEM); + } + + return link_tbl; +} + +static void free_link_table_buf(struct udma_dev *udma_dev, + struct udma_link_table *tbl) +{ + if (tbl->buf) { + uint32_t size = tbl->buf->npages * sizeof(uint64_t); + + dma_free_coherent(udma_dev->dev, size, tbl->table.buf, + tbl->table.map); + } + + udma_buf_free(udma_dev, tbl->buf); +} + +static int udma_init_link_table(struct udma_dev *udma_dev) +{ + struct udma_link_table *link_tbl; + int ret; + + link_tbl = alloc_link_table_buf(udma_dev); + if (IS_ERR(link_tbl)) + return -ENOMEM; + + if (WARN_ON(link_tbl->buf->npages > UDMA_EXT_LLM_MAX_DEPTH)) { + ret = -EINVAL; + goto err_alloc; + } + + config_llm_table(link_tbl->buf, link_tbl->table.buf); + ret = set_llm_cfg_to_hw(udma_dev, link_tbl); + if (ret) + goto err_alloc; + + return 0; + +err_alloc: + free_link_table_buf(udma_dev, link_tbl); + return ret; +} + +static void udma_free_link_table(struct udma_dev *udma_dev) +{ + struct udma_priv *priv = (struct udma_priv *)udma_dev->priv; + + free_link_table_buf(udma_dev, &priv->ext_llm); +} + +static int udma_clear_extdb_list_info(struct udma_dev *udma_dev) +{ + struct udma_cmq_desc desc; + int ret; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_CLEAR_EXTDB_LIST_INFO, + false); + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) + dev_err(udma_dev->dev, + "failed to clear extended doorbell info, ret = %d.\n", + ret); + + return ret; +} + +int get_hem_table(struct udma_dev *udma_dev) +{ + uint32_t qpc_count; + uint32_t cqc_count; + uint32_t gmv_count; + uint32_t i; + int ret; + + /* Alloc memory for source address table buffer space chunk */ + for (gmv_count = 0; gmv_count < udma_dev->caps.gmv_entry_num; + gmv_count++) { + ret = udma_table_get(udma_dev, &udma_dev->gmv_table, gmv_count); + if (ret) + goto err_gmv_failed; + } + + /* Alloc memory for QPC Timer buffer space chunk */ + for (qpc_count = 0; qpc_count < udma_dev->caps.qpc_timer_bt_num; + qpc_count++) { + ret = udma_table_get(udma_dev, &udma_dev->qpc_timer_table, + qpc_count); + if (ret) { + dev_err(udma_dev->dev, "QPC Timer get failed\n"); + goto err_qpc_timer_failed; + } + } + + /* Alloc memory for CQC Timer buffer space chunk */ + for (cqc_count = 0; cqc_count < udma_dev->caps.cqc_timer_bt_num; + cqc_count++) { + ret = udma_table_get(udma_dev, &udma_dev->cqc_timer_table, + cqc_count); + if (ret) { + dev_err(udma_dev->dev, "CQC Timer get failed\n"); + goto err_cqc_timer_failed; + } + } + + return 0; + +err_cqc_timer_failed: + for (i = 0; i < cqc_count; i++) + udma_table_put(udma_dev, &udma_dev->cqc_timer_table, i); + +err_qpc_timer_failed: + for (i = 0; i < qpc_count; i++) + udma_table_put(udma_dev, &udma_dev->qpc_timer_table, i); + +err_gmv_failed: + for (i = 0; i < gmv_count; i++) + udma_table_put(udma_dev, &udma_dev->gmv_table, i); + + return ret; +} + +void put_hem_table(struct udma_dev *udma_dev) +{ + uint32_t i; + + for (i = 0; i < udma_dev->caps.gmv_entry_num; i++) + udma_table_put(udma_dev, &udma_dev->gmv_table, i); + + + for (i = 0; i < udma_dev->caps.qpc_timer_bt_num; i++) + udma_table_put(udma_dev, &udma_dev->qpc_timer_table, i); + + for (i = 0; i < udma_dev->caps.cqc_timer_bt_num; i++) + udma_table_put(udma_dev, &udma_dev->cqc_timer_table, i); +} + static int udma_hw_init(struct udma_dev *udma_dev) { + int ret; + + /* UDMA requires the extdb info to be cleared before using */ + ret = udma_clear_extdb_list_info(udma_dev); + if (ret) + goto err_clear_extdb_failed; + + ret = get_hem_table(udma_dev); + if (ret) + goto err_clear_extdb_failed; + + ret = udma_init_link_table(udma_dev); + if (ret) { + dev_err(udma_dev->dev, "failed to init llm, ret = %d.\n", ret); + goto err_llm_init_failed; + } + return 0; + +err_llm_init_failed: + put_hem_table(udma_dev); +err_clear_extdb_failed: + + return ret; } static void udma_hw_exit(struct udma_dev *udma_dev) { + udma_free_link_table(udma_dev); + + put_hem_table(udma_dev); +} + +static int get_op_for_set_hem(uint32_t type, int step_idx, uint16_t *mbox_op, + bool is_create) +{ + uint16_t op; + + switch (type) { + case HEM_TYPE_QPC: + op = is_create ? UDMA_CMD_WRITE_QPC_BT0 : + UDMA_CMD_DESTROY_QPC_BT0; + break; + case HEM_TYPE_MTPT: + op = is_create ? UDMA_CMD_WRITE_MPT_BT0 : + UDMA_CMD_DESTROY_MPT_BT0; + break; + case HEM_TYPE_CQC: + op = is_create ? UDMA_CMD_WRITE_CQC_BT0 : + UDMA_CMD_DESTROY_CQC_BT0; + break; + case HEM_TYPE_SRQC: + op = is_create ? UDMA_CMD_WRITE_SRQC_BT0 : + UDMA_CMD_DESTROY_SRQC_BT0; + break; + case HEM_TYPE_SCCC: + op = is_create ? UDMA_CMD_WRITE_SCCC_BT0 : UDMA_CMD_RESERVED; + break; + case HEM_TYPE_QPC_TIMER: + op = is_create ? UDMA_CMD_WRITE_QPC_TIMER_BT0 : + UDMA_CMD_RESERVED; + break; + case HEM_TYPE_CQC_TIMER: + op = is_create ? UDMA_CMD_WRITE_CQC_TIMER_BT0 : + UDMA_CMD_RESERVED; + break; + case HEM_TYPE_GMV: + op = is_create ? UDMA_CMD_RESERVED : UDMA_CMD_RESERVED; + break; + default: + return -EINVAL; + } + + if (op != UDMA_CMD_RESERVED) + *mbox_op = op + step_idx; + + return 0; +} + +static int config_gmv_ba_to_hw(struct udma_dev *udma_dev, uint64_t obj, + dma_addr_t base_addr) +{ + struct udma_cmq_desc desc; + struct udma_cmq_req *req = (struct udma_cmq_req *)desc.data; + uint32_t idx = obj / (UDMA_PAGE_SIZE / udma_dev->caps.gmv_entry_sz); + uint64_t addr = to_hr_hw_page_addr(base_addr); + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_CFG_GMV_BT, false); + + udma_reg_write(req, CFG_GMV_BT_BA_L, lower_32_bits(addr)); + udma_reg_write(req, CFG_GMV_BT_BA_H, upper_32_bits(addr)); + udma_reg_write(req, CFG_GMV_BT_IDX, idx); + udma_reg_write(req, CFG_GMV_BT_VF_ID, 0); + + return udma_cmq_send(udma_dev, &desc, 1); +} + +static int config_hem_ba_to_hw(struct udma_dev *udma_dev, uint64_t obj, + uint64_t base_addr, uint16_t op) +{ + struct udma_cmd_mailbox *mbox = udma_alloc_cmd_mailbox(udma_dev); + struct udma_cmq_desc desc; + struct udma_mbox *mb; + int ret; + + if (IS_ERR_OR_NULL(mbox)) + return -ENOMEM; + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mbox_desc_init(mb, base_addr, mbox->dma, obj, op); + ret = udma_cmd_mbox(udma_dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); + if (ret) + dev_err(udma_dev->dev, "[mailbox cmd] config hem ba failed.\n"); + + udma_free_cmd_mailbox(udma_dev, mbox); + + return ret; +} + +static int set_hem_to_hw(struct udma_dev *udma_dev, int obj, + dma_addr_t base_addr, + uint32_t hem_type, int step_idx) +{ + bool is_create = true; + uint16_t op; + int ret; + + if (unlikely(hem_type == HEM_TYPE_GMV)) + return config_gmv_ba_to_hw(udma_dev, obj, base_addr); + + if (unlikely(hem_type == HEM_TYPE_SCCC && step_idx)) + return 0; + + ret = get_op_for_set_hem(hem_type, step_idx, &op, is_create); + if (ret < 0) + return ret; + return config_hem_ba_to_hw(udma_dev, obj, base_addr, op); +} + +static int udma_set_hem(struct udma_dev *udma_dev, struct udma_hem_table *table, + int obj, int step_idx) +{ + struct udma_hem_iter iter; + struct udma_hem_mhop mhop; + uint64_t mhop_obj = obj; + uint32_t chunk_ba_num; + struct udma_hem *hem; + uint64_t hem_idx = 0; + uint64_t l1_idx = 0; + uint64_t bt_ba = 0; + uint32_t hop_num; + int i, j, k; + int ret; + + if (!udma_check_whether_mhop(udma_dev, table->type)) + return 0; + + udma_calc_hem_mhop(udma_dev, table, &mhop_obj, &mhop); + i = mhop.l0_idx; + j = mhop.l1_idx; + k = mhop.l2_idx; + hop_num = mhop.hop_num; + chunk_ba_num = mhop.bt_chunk_size / BA_BYTE_LEN; + + if (hop_num == 2) { + hem_idx = i * chunk_ba_num * chunk_ba_num + j * chunk_ba_num + + k; + l1_idx = i * chunk_ba_num + j; + } else if (hop_num == 1) { + hem_idx = i * chunk_ba_num + j; + } else if (hop_num == UDMA_HOP_NUM_0) { + hem_idx = i; + } + + if (table->type == HEM_TYPE_SCCC) + obj = mhop.l0_idx; + + if (check_whether_last_step(hop_num, step_idx)) { + hem = table->hem[hem_idx]; + for (udma_hem_first(hem, &iter); + !udma_hem_last(&iter); udma_hem_next(&iter)) { + bt_ba = udma_hem_addr(&iter); + ret = set_hem_to_hw(udma_dev, obj, bt_ba, table->type, + step_idx); + } + } else { + if (step_idx == 0) + bt_ba = table->bt_l0_dma_addr[i]; + else if (step_idx == 1 && hop_num == 2) + bt_ba = table->bt_l1_dma_addr[l1_idx]; + + ret = set_hem_to_hw(udma_dev, obj, bt_ba, table->type, + step_idx); + } + + return ret; +} + +static int udma_clear_hem(struct udma_dev *udma_dev, + struct udma_hem_table *table, + int obj, int step_idx) +{ + uint16_t op = UDMA_CMD_RESERVED; + bool is_create = false; + int ret; + + if (!udma_check_whether_mhop(udma_dev, table->type)) + return 0; + + ret = get_op_for_set_hem(table->type, step_idx, &op, is_create); + if (ret < 0 || op == UDMA_CMD_RESERVED) + return ret; + + return config_hem_ba_to_hw(udma_dev, obj, 0, op); } static const struct udma_hw udma_hw = { @@ -496,8 +1441,28 @@ static const struct udma_hw udma_hw = { .hw_exit = udma_hw_exit, .post_mbox = udma_post_mbox, .poll_mbox_done = udma_poll_mbox_done, + .set_hem = udma_set_hem, + .clear_hem = udma_clear_hem, }; +bool udma_is_virtual_func(struct pci_dev *pdev) +{ + uint32_t dev_id = pdev->device; + + switch (dev_id) { + case HNAE3_DEV_ID_UDMA_OVER_UBL: + case HNAE3_DEV_ID_UDMA: + return false; + case HNAE3_DEV_ID_UDMA_OVER_UBL_VF: + return true; + default: + dev_warn(&pdev->dev, "un-recognized pci deviced-id %x.\n", + dev_id); + } + + return false; +} + static void udma_get_cfg(struct udma_dev *udma_dev, struct hnae3_handle *handle) { diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.h b/drivers/ub/hw/hns3/hns3_udma_hw.h index 2f232a264f45..19b69496d82e 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hw.h +++ b/drivers/ub/hw/hns3/hns3_udma_hw.h @@ -58,6 +58,29 @@ struct udma_hw_id_query_cmq { uint32_t rsv[4]; }; +struct udma_query_oor_cmq { + uint8_t oor_en; + uint8_t reorder_cq_buffer_en; + uint8_t reorder_cap; + uint8_t reorder_cq_shift; + uint32_t on_flight_size; + uint8_t dynamic_ack_timeout; + uint8_t rsv[15]; +}; + +#define UDMA_MAX_MTT_SEGS 0x1000000 +#define UDMA_MAX_SRQWQE_SEGS 0x1000000 +#define UDMA_MAX_IDX_SEGS 0x1000000 +#define UDMA_MTT_ENTRY_SZ 64 +#define UDMA_EID_SIZE_IDX 4 + +#define UDMA_PBL_HOP_NUM 2 +#define UDMA_EQE_HOP_NUM 2 + +#define UDMA_QPC_TIMER_ENTRY_SZ PAGE_SIZE +#define UDMA_CQC_TIMER_ENTRY_SZ PAGE_SIZE + +#define UDMA_EQE_SIZE 0x40 #define NIC_ICL_SWITCH_CMD_UDMA_SEL_SHIFT 0 #define NIC_ICL_SWITCH_CMD_UDMA_SEL BIT(NIC_ICL_SWITCH_CMD_UDMA_SEL_SHIFT) @@ -74,7 +97,67 @@ struct udma_hw_id_query_cmq { #define CFG_GLOBAL_PARAM_1US_CYCLES CMQ_REQ_FIELD_LOC(9, 0) #define CFG_GLOBAL_PARAM_UDP_PORT CMQ_REQ_FIELD_LOC(31, 16) +#define UDMA_FUNC_IRQ_RSV 2 #define UDMA_1US_CFG 999 +#define UDMA_EXT_LLM_ENTRY_SZ 8 +#define UDMA_EXT_LLM_MAX_DEPTH 4096 + +#define UDMA_BA_PG_SZ_SUPPORTED_256K 6 +#define UDMA_BA_PG_SZ_SUPPORTED_16K 2 + +/* Fields of UDMA_OPC_EXT_CFG */ +#define EXT_CFG_VF_ID CMQ_REQ_FIELD_LOC(31, 0) +#define EXT_CFG_QP_PI_INDEX CMQ_REQ_FIELD_LOC(45, 32) +#define EXT_CFG_QP_PI_NUM CMQ_REQ_FIELD_LOC(63, 48) +#define EXT_CFG_QP_NUM CMQ_REQ_FIELD_LOC(87, 64) +#define EXT_CFG_QP_INDEX CMQ_REQ_FIELD_LOC(119, 96) +#define EXT_CFG_LLM_IDX CMQ_REQ_FIELD_LOC(139, 128) +#define EXT_CFG_LLM_NUM CMQ_REQ_FIELD_LOC(156, 144) + +#define CFG_LLM_A_BA_L CMQ_REQ_FIELD_LOC(31, 0) +#define CFG_LLM_A_BA_H CMQ_REQ_FIELD_LOC(63, 32) +#define CFG_LLM_A_DEPTH CMQ_REQ_FIELD_LOC(76, 64) +#define CFG_LLM_A_PG_SZ CMQ_REQ_FIELD_LOC(83, 80) +#define CFG_LLM_A_INIT_EN CMQ_REQ_FIELD_LOC(84, 84) +#define CFG_LLM_A_HEAD_BA_L CMQ_REQ_FIELD_LOC(127, 96) +#define CFG_LLM_A_HEAD_BA_H CMQ_REQ_FIELD_LOC(147, 128) +#define CFG_LLM_A_HEAD_NXT_PTR CMQ_REQ_FIELD_LOC(159, 148) +#define CFG_LLM_A_HEAD_PTR CMQ_REQ_FIELD_LOC(171, 160) +#define CFG_LLM_B_TAIL_BA_L CMQ_REQ_FIELD_LOC(31, 0) +#define CFG_LLM_B_TAIL_BA_H CMQ_REQ_FIELD_LOC(63, 32) +#define CFG_LLM_B_TAIL_PTR CMQ_REQ_FIELD_LOC(75, 64) + +/* + * Fields of UDMA_OPC_QUERY_PF_RES, UDMA_OPC_QUERY_VF_RES + * and UDMA_OPC_ALLOC_VF_RES + */ +#define FUNC_RES_A_VF_ID CMQ_REQ_FIELD_LOC(7, 0) +#define FUNC_RES_A_QPC_BT_INDEX CMQ_REQ_FIELD_LOC(42, 32) +#define FUNC_RES_A_QPC_BT_NUM CMQ_REQ_FIELD_LOC(59, 48) +#define FUNC_RES_A_SRQC_BT_INDEX CMQ_REQ_FIELD_LOC(72, 64) +#define FUNC_RES_A_SRQC_BT_NUM CMQ_REQ_FIELD_LOC(89, 80) +#define FUNC_RES_A_CQC_BT_INDEX CMQ_REQ_FIELD_LOC(104, 96) +#define FUNC_RES_A_CQC_BT_NUM CMQ_REQ_FIELD_LOC(121, 112) +#define FUNC_RES_A_MPT_BT_INDEX CMQ_REQ_FIELD_LOC(136, 128) +#define FUNC_RES_A_MPT_BT_NUM CMQ_REQ_FIELD_LOC(153, 144) +#define FUNC_RES_A_EQC_BT_INDEX CMQ_REQ_FIELD_LOC(168, 160) +#define FUNC_RES_A_EQC_BT_NUM CMQ_REQ_FIELD_LOC(185, 176) +#define FUNC_RES_B_SMAC_NUM CMQ_REQ_FIELD_LOC(48, 40) +#define FUNC_RES_B_SGID_NUM CMQ_REQ_FIELD_LOC(80, 72) +#define FUNC_RES_B_QID_INDEX CMQ_REQ_FIELD_LOC(105, 96) +#define FUNC_RES_B_QID_NUM CMQ_REQ_FIELD_LOC(122, 112) +#define FUNC_RES_V_QID_NUM CMQ_REQ_FIELD_LOC(115, 112) + +#define FUNC_RES_B_SCCC_BT_INDEX CMQ_REQ_FIELD_LOC(136, 128) +#define FUNC_RES_B_SCCC_BT_NUM CMQ_REQ_FIELD_LOC(145, 137) +#define FUNC_RES_B_GMV_BT_INDEX CMQ_REQ_FIELD_LOC(167, 160) +#define FUNC_RES_B_GMV_BT_NUM CMQ_REQ_FIELD_LOC(176, 168) +#define FUNC_RES_V_GMV_BT_NUM CMQ_REQ_FIELD_LOC(184, 176) + +/* Fields of UDMA_OPC_QUERY_PF_TIMER_RES */ +#define PF_TIMER_RES_QPC_ITEM_NUM CMQ_REQ_FIELD_LOC(60, 48) +#define PF_TIMER_RES_CQC_ITEM_NUM CMQ_REQ_FIELD_LOC(91, 80) + enum { UDMA_CMD_FLAG_IN = BIT(0), @@ -94,10 +177,23 @@ enum udma_opcode_type { UDMA_OPC_QUERY_HW_ID = 0x7032, UDMA_OPC_QUERY_HW_VER = 0x8000, UDMA_OPC_CFG_GLOBAL_PARAM = 0x8001, + UDMA_OPC_QUERY_PF_RES = 0x8400, + UDMA_OPC_ALLOC_VF_RES = 0x8401, + UDMA_OPC_CFG_EXT_LLM = 0x8403, + UDMA_OPC_QUERY_PF_TIMER_RES = 0x8406, UDMA_OPC_QUERY_FUNC_INFO = 0x8407, UDMA_OPC_QUERY_PF_CAPS_NUM = 0x8408, + UDMA_OPC_CFG_ENTRY_SIZE = 0x8409, + UDMA_OPC_QUERY_VF_CAPS_NUM = 0x8410, + UDMA_OPC_POST_MB = 0x8504, UDMA_OPC_QUERY_MB_ST = 0x8505, + UDMA_OPC_CFG_BT_ATTR = 0x8506, + UDMA_OPC_CLEAR_EXTDB_LIST_INFO = 0x850d, + UDMA_OPC_QUERY_VF_RES = 0x850e, + UDMA_OPC_CFG_GMV_BT = 0x8510, + UDMA_OPC_EXT_CFG = 0x8512, UDMA_SWITCH_PARAMETER_CFG = 0x1033, + UDMA_QUERY_OOR_CAPS = 0xA002, }; #define UDMA_QUERY_PF_CAPS_CMD_NUM 5 @@ -171,6 +267,9 @@ struct udma_query_pf_caps_e { uint16_t aeq_period; }; +#define UDMA_EXT_LLM_ENTRY(addr, id) (((id) << (64 - 12)) | ((addr) >> 12)) +#define UDMA_EXT_LLM_MIN_PAGES(que_num) ((que_num) * 4 + 2) + #define QUERY_PF_CAPS_C_NUM_PDS_S 0 #define QUERY_PF_CAPS_C_NUM_PDS_M GENMASK(19, 0) @@ -260,4 +359,35 @@ struct udma_query_pf_caps_e { #define QUERY_PF_CAPS_E_RSV_LKEYS_S 0 #define QUERY_PF_CAPS_E_RSV_LKEYS_M GENMASK(19, 0) +/* Fields of UDMA_OPC_CFG_BT_ATTR */ +#define CFG_BT_QPC_BA_PGSZ CMQ_REQ_FIELD_LOC(3, 0) +#define CFG_BT_QPC_BUF_PGSZ CMQ_REQ_FIELD_LOC(7, 4) +#define CFG_BT_QPC_HOPNUM CMQ_REQ_FIELD_LOC(9, 8) +#define CFG_BT_SRQC_BA_PGSZ CMQ_REQ_FIELD_LOC(35, 32) +#define CFG_BT_SRQC_BUF_PGSZ CMQ_REQ_FIELD_LOC(39, 36) +#define CFG_BT_SRQC_HOPNUM CMQ_REQ_FIELD_LOC(41, 40) +#define CFG_BT_CQC_BA_PGSZ CMQ_REQ_FIELD_LOC(67, 64) +#define CFG_BT_CQC_BUF_PGSZ CMQ_REQ_FIELD_LOC(71, 68) +#define CFG_BT_CQC_HOPNUM CMQ_REQ_FIELD_LOC(73, 72) +#define CFG_BT_MPT_BA_PGSZ CMQ_REQ_FIELD_LOC(99, 96) +#define CFG_BT_MPT_BUF_PGSZ CMQ_REQ_FIELD_LOC(103, 100) +#define CFG_BT_MPT_HOPNUM CMQ_REQ_FIELD_LOC(105, 104) +#define CFG_BT_SCCC_BA_PGSZ CMQ_REQ_FIELD_LOC(131, 128) +#define CFG_BT_SCCC_BUF_PGSZ CMQ_REQ_FIELD_LOC(135, 132) +#define CFG_BT_SCCC_HOPNUM CMQ_REQ_FIELD_LOC(137, 136) + +/* Fields of UDMA_OPC_CFG_ENTRY_SIZE */ +#define CFG_HEM_ENTRY_SIZE_TYPE CMQ_REQ_FIELD_LOC(31, 0) +enum { + UDMA_CFG_QPC_SIZE = BIT(0), + UDMA_CFG_SCCC_SIZE = BIT(1), +}; + +#define CFG_HEM_ENTRY_SIZE_VALUE CMQ_REQ_FIELD_LOC(191, 160) + +#define CFG_GMV_BT_BA_L CMQ_REQ_FIELD_LOC(31, 0) +#define CFG_GMV_BT_BA_H CMQ_REQ_FIELD_LOC(51, 32) +#define CFG_GMV_BT_IDX CMQ_REQ_FIELD_LOC(95, 64) +#define CFG_GMV_BT_VF_ID CMQ_REQ_FIELD_LOC(103, 96) + #endif /* _UDMA_HW_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c index 46ad4d75b628..268094b1e25a 100644 --- a/drivers/ub/hw/hns3/hns3_udma_main.c +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -13,14 +13,204 @@ * */ -#include "hns3_udma_device.h" #include "urma/ubcore_api.h" #include "hns3_udma_device.h" +#include "hns3_udma_hem.h" +#include "hns3_udma_cmd.h" static struct ubcore_ops g_udma_dev_ops = { }; +int udma_init_common_hem(struct udma_dev *udma_dev) +{ + struct device *dev = udma_dev->dev; + int ret; + + ret = udma_init_hem_table(udma_dev, &udma_dev->seg_table.table, + HEM_TYPE_MTPT, udma_dev->caps.mtpt_entry_sz, + udma_dev->caps.num_mtpts); + if (ret) { + dev_err(dev, "Failed to init MTPT context memory.\n"); + return ret; + } + dev_info(dev, "init MPT hem table success.\n"); + + ret = udma_init_hem_table(udma_dev, &udma_dev->qp_table.qp_table, + HEM_TYPE_QPC, udma_dev->caps.qpc_sz, + udma_dev->caps.num_qps); + if (ret) { + dev_err(dev, "Failed to init QP context memory.\n"); + goto err_unmap_dmpt; + } + dev_info(dev, "init QPC hem table success.\n"); + + ret = udma_init_hem_table(udma_dev, &udma_dev->qp_table.irrl_table, + HEM_TYPE_IRRL, udma_dev->caps.irrl_entry_sz * + udma_dev->caps.max_qp_init_rdma, + udma_dev->caps.num_qps); + if (ret) { + dev_err(dev, "Failed to init irrl_table memory.\n"); + goto err_unmap_qp; + } + dev_info(dev, "init IRRL hem table success.\n"); + + if (udma_dev->caps.trrl_entry_sz) { + ret = udma_init_hem_table(udma_dev, + &udma_dev->qp_table.trrl_table, + HEM_TYPE_TRRL, + udma_dev->caps.trrl_entry_sz * + udma_dev->caps.max_qp_dest_rdma, + udma_dev->caps.num_qps); + if (ret) { + dev_err(dev, "Failed to init trrl_table memory.\n"); + goto err_unmap_irrl; + } + dev_info(dev, "init TRRL hem table success.\n"); + } + + ret = udma_init_hem_table(udma_dev, &udma_dev->jfc_table.table, + HEM_TYPE_CQC, udma_dev->caps.cqc_entry_sz, + udma_dev->caps.num_cqs); + if (ret) { + dev_err(dev, "Failed to init CQ context memory.\n"); + goto err_unmap_trrl; + } + dev_info(dev, "init CQC hem table success.\n"); + + if (udma_dev->caps.flags & UDMA_CAP_FLAG_SRQ) { + ret = udma_init_hem_table(udma_dev, &udma_dev->jfr_table.table, + HEM_TYPE_SRQC, + udma_dev->caps.srqc_entry_sz, + udma_dev->caps.num_srqs); + if (ret) { + dev_err(dev, "Failed to init SRQ context memory.\n"); + goto err_unmap_cq; + } + dev_info(dev, "init SRQC hem table success.\n"); + } + + if (udma_dev->caps.flags & UDMA_CAP_FLAG_QP_FLOW_CTRL) { + ret = udma_init_hem_table(udma_dev, + &udma_dev->qp_table.sccc_table, + HEM_TYPE_SCCC, + udma_dev->caps.scc_ctx_sz, + udma_dev->caps.num_qps); + if (ret) { + dev_err(dev, "Failed to init SCC context memory.\n"); + goto err_unmap_srq; + } + dev_info(dev, "init SCCC hem table success.\n"); + } + + if (udma_dev->caps.gmv_entry_sz) { + ret = udma_init_hem_table(udma_dev, &udma_dev->gmv_table, + HEM_TYPE_GMV, + udma_dev->caps.gmv_entry_sz, + udma_dev->caps.gmv_entry_num); + if (ret) { + dev_err(dev, "failed to init gmv table memory.\n"); + goto err_unmap_ctx; + } + dev_info(dev, "init GMV hem table success.\n"); + } + + return 0; +err_unmap_ctx: + if (udma_dev->caps.flags & UDMA_CAP_FLAG_QP_FLOW_CTRL) + udma_cleanup_hem_table(udma_dev, + &udma_dev->qp_table.sccc_table); +err_unmap_srq: + if (udma_dev->caps.flags & UDMA_CAP_FLAG_SRQ) + udma_cleanup_hem_table(udma_dev, &udma_dev->jfr_table.table); +err_unmap_cq: + udma_cleanup_hem_table(udma_dev, &udma_dev->jfc_table.table); +err_unmap_trrl: + if (udma_dev->caps.trrl_entry_sz) + udma_cleanup_hem_table(udma_dev, + &udma_dev->qp_table.trrl_table); +err_unmap_irrl: + udma_cleanup_hem_table(udma_dev, &udma_dev->qp_table.irrl_table); +err_unmap_qp: + udma_cleanup_hem_table(udma_dev, &udma_dev->qp_table.qp_table); +err_unmap_dmpt: + udma_cleanup_hem_table(udma_dev, &udma_dev->seg_table.table); + + return ret; +} + +static int udma_init_hem(struct udma_dev *udma_dev) +{ + struct device *dev = udma_dev->dev; + int ret; + + ret = udma_init_common_hem(udma_dev); + if (ret) { + dev_err(dev, "Failed to init common hem table of PF.\n"); + return ret; + } + + if (udma_dev->caps.qpc_timer_entry_sz) { + ret = udma_init_hem_table(udma_dev, &udma_dev->qpc_timer_table, + HEM_TYPE_QPC_TIMER, + udma_dev->caps.qpc_timer_entry_sz, + udma_dev->caps.num_qpc_timer); + if (ret) { + dev_err(dev, "Failed to init QPC timer memory.\n"); + goto err_unmap_vf_hem; + } + } + if (udma_dev->caps.cqc_timer_entry_sz) { + ret = udma_init_hem_table(udma_dev, &udma_dev->cqc_timer_table, + HEM_TYPE_CQC_TIMER, + udma_dev->caps.cqc_timer_entry_sz, + udma_dev->caps.cqc_timer_bt_num); + if (ret) { + dev_err(dev, "Failed to init CQC timer memory.\n"); + goto err_unmap_qpc_timer; + } + } + + return 0; +err_unmap_qpc_timer: + if (udma_dev->caps.qpc_timer_entry_sz) + udma_cleanup_hem_table(udma_dev, &udma_dev->qpc_timer_table); +err_unmap_vf_hem: + udma_cleanup_common_hem(udma_dev); + + return ret; +} + +void udma_cleanup_common_hem(struct udma_dev *udma_dev) +{ + if (udma_dev->caps.gmv_entry_sz) + udma_cleanup_hem_table(udma_dev, &udma_dev->gmv_table); + if (udma_dev->caps.flags & UDMA_CAP_FLAG_QP_FLOW_CTRL) + udma_cleanup_hem_table(udma_dev, + &udma_dev->qp_table.sccc_table); + if (udma_dev->caps.flags & UDMA_CAP_FLAG_SRQ) + udma_cleanup_hem_table(udma_dev, &udma_dev->jfr_table.table); + udma_cleanup_hem_table(udma_dev, &udma_dev->jfc_table.table); + if (udma_dev->caps.trrl_entry_sz) + udma_cleanup_hem_table(udma_dev, + &udma_dev->qp_table.trrl_table); + + udma_cleanup_hem_table(udma_dev, &udma_dev->qp_table.irrl_table); + udma_cleanup_hem_table(udma_dev, &udma_dev->qp_table.qp_table); + udma_cleanup_hem_table(udma_dev, &udma_dev->seg_table.table); +} + +static void udma_cleanup_hem(struct udma_dev *udma_dev) +{ + if (udma_dev->caps.qpc_timer_entry_sz) + udma_cleanup_hem_table(udma_dev, &udma_dev->qpc_timer_table); + if (udma_dev->caps.cqc_timer_entry_sz) + udma_cleanup_hem_table(udma_dev, &udma_dev->cqc_timer_table); + + udma_cleanup_common_hem(udma_dev); +} + + static void udma_set_devname(struct udma_dev *udma_dev, struct ubcore_device *ub_dev) { @@ -89,6 +279,12 @@ int udma_hnae_client_init(struct udma_dev *udma_dev) } } + ret = udma_init_hem(udma_dev); + if (ret) { + dev_err(dev, "init HEM(Hardware Entry Memory) failed!\n"); + goto error_failed_hem_init; + } + ret = udma_dev->hw->hw_init(udma_dev); if (ret) { dev_err(dev, "hw_init failed!\n"); @@ -107,6 +303,14 @@ int udma_hnae_client_init(struct udma_dev *udma_dev) udma_dev->hw->hw_exit(udma_dev); error_failed_engine_init: + udma_cleanup_hem(udma_dev); + +error_failed_hem_init: + if (udma_dev->cmd_mod) + udma_cmd_use_polling(udma_dev); + + udma_cmd_cleanup(udma_dev); + error_failed_cmd_init: error_failed_hw_profile: udma_dev->hw->cmq_exit(udma_dev); -- Gitee From e1d0edff5045dc6b9abbe2b1d14aa2c2b9d2028b Mon Sep 17 00:00:00 2001 From: Weibo Zhao Date: Fri, 11 Aug 2023 10:43:26 +0800 Subject: [PATCH 04/42] hns3 udma: feature of memory translate region driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA ------------------------------------------------------ This patch is the code of Memory-Translate-Region(MTR). MTR is a way that hardware can read/write memory. Signed-off-by: Weibo Zhao --- drivers/ub/hw/hns3/hns3_udma_device.h | 52 ++ drivers/ub/hw/hns3/hns3_udma_hem.c | 1108 +++++++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_hem.h | 17 + 3 files changed, 1177 insertions(+) diff --git a/drivers/ub/hw/hns3/hns3_udma_device.h b/drivers/ub/hw/hns3/hns3_udma_device.h index d668e55bde14..7bc8b374a92b 100644 --- a/drivers/ub/hw/hns3/hns3_udma_device.h +++ b/drivers/ub/hw/hns3/hns3_udma_device.h @@ -55,6 +55,9 @@ #define UDMA_TX_CMQ_PI_REG 0x07010 #define UDMA_TX_CMQ_CI_REG 0x07014 +#define UDMA_MAX_BT_REGION 3 +#define UDMA_MAX_BT_LEVEL 3 + enum udma_reset_stage { UDMA_STATE_RST_DOWN = 2, UDMA_STATE_RST_UNINIT, @@ -104,11 +107,47 @@ struct udma_ida { uint32_t min; /* Lowest ID to allocate. */ uint32_t max; /* Highest ID to allocate. */ }; + +struct udma_buf_region { + uint32_t offset; /* page offset */ + uint32_t count; /* page count */ + int hopnum; /* addressing hop num */ +}; + +struct udma_hem_list { + struct list_head root_bt; + /* link all bt dma mem by hop config */ + struct list_head mid_bt[UDMA_MAX_BT_REGION][UDMA_MAX_BT_LEVEL]; + struct list_head btm_bt; /* link all bottom bt in @mid_bt */ + dma_addr_t root_ba; /* pointer to the root ba table */ +}; + +struct udma_buf_attr { + struct { + size_t size; /* region size */ + int hopnum; /* multi-hop addressing hop num */ + } region[UDMA_MAX_BT_REGION]; + uint32_t region_count; /* valid region count */ + uint32_t page_shift; /* buffer page shift */ + /* only alloc buffer-required MTT memory */ + bool mtt_only; +}; + struct udma_buf_list { void *buf; dma_addr_t map; }; +struct udma_hem_cfg { + dma_addr_t root_ba; /* root BA table's address */ + bool is_direct; /* addressing without BA table */ + uint32_t ba_pg_shift; /* BA table page shift */ + uint32_t buf_pg_shift; /* buffer page shift */ + uint32_t buf_pg_count; /* buffer page count */ + struct udma_buf_region region[UDMA_MAX_BT_REGION]; + uint32_t region_count; +}; + struct udma_buf { struct udma_buf_list *trunk_list; uint32_t ntrunks; @@ -121,6 +160,15 @@ struct udma_link_table { struct udma_buf_list table; struct udma_buf *buf; }; + +/* memory translate region */ +struct udma_mtr { + struct udma_hem_list hem_list; /* multi-hop addressing resource */ + struct ubcore_umem *umem; /* user space buffer */ + struct udma_buf *kmem; /* kernel space buffer */ + struct udma_hem_cfg hem_cfg; /* config for hardware addressing */ +}; + struct udma_dev; struct udma_cmd_context { struct completion done; @@ -498,6 +546,10 @@ int udma_cmq_send(struct udma_dev *udma_dev, struct udma_cmq_desc *desc, int num); int udma_hnae_client_init(struct udma_dev *udma_dev); void udma_hnae_client_exit(struct udma_dev *udma_dev); +int udma_mtr_create(struct udma_dev *udma_dev, struct udma_mtr *mtr, + struct udma_buf_attr *buf_attr, uint32_t ba_page_shift, + uint64_t user_addr, bool is_user); +void udma_mtr_destroy(struct udma_dev *udma_dev, struct udma_mtr *mtr); void udma_cleanup_common_hem(struct udma_dev *udma_dev); int udma_init_common_hem(struct udma_dev *udma_dev); diff --git a/drivers/ub/hw/hns3/hns3_udma_hem.c b/drivers/ub/hw/hns3/hns3_udma_hem.c index 34284c993b59..e425efaa258f 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hem.c +++ b/drivers/ub/hw/hns3/hns3_udma_hem.c @@ -676,6 +676,75 @@ void udma_table_put(struct udma_dev *udma_dev, mutex_unlock(&table->mutex); } +void *udma_table_find(struct udma_dev *udma_dev, + struct udma_hem_table *table, + uint64_t obj, dma_addr_t *dma_handle) +{ + struct udma_hem_chunk *chunk; + struct udma_hem_mhop mhop; + uint64_t mhop_obj = obj; + uint64_t obj_per_chunk; + int offset, dma_offset; + struct udma_hem *hem; + uint32_t hem_idx = 0; + uint64_t idx_offset; + void *addr = NULL; + uint32_t length; + uint32_t i, j; + + mutex_lock(&table->mutex); + + if (!udma_check_whether_mhop(udma_dev, table->type)) { + obj_per_chunk = table->table_chunk_size / table->obj_size; + hem = table->hem[obj / obj_per_chunk]; + idx_offset = obj % obj_per_chunk; + dma_offset = offset = idx_offset * table->obj_size; + } else { + /* 8 bytes per BA and 8 BA per segment */ + uint32_t seg_size = 64; + + if (udma_calc_hem_mhop(udma_dev, table, &mhop_obj, &mhop)) + goto out; + /* mtt mhop */ + i = mhop.l0_idx; + j = mhop.l1_idx; + if (mhop.hop_num == 2) + hem_idx = i * (mhop.bt_chunk_size / BA_BYTE_LEN) + j; + else if (mhop.hop_num == 1 || + mhop.hop_num == UDMA_HOP_NUM_0) + hem_idx = i; + + hem = table->hem[hem_idx]; + dma_offset = offset = obj * seg_size % mhop.bt_chunk_size; + if (mhop.hop_num == 2) + dma_offset = offset = 0; + } + + if (!hem) + goto out; + + list_for_each_entry(chunk, &hem->chunk_list, list) { + for (i = 0; (int)i < chunk->npages; ++i) { + length = sg_dma_len(&chunk->mem[i]); + if (dma_handle && dma_offset >= 0) { + *dma_handle = length > (uint32_t)dma_offset ? sg_dma_address( + &chunk->mem[i]) + dma_offset : *dma_handle; + dma_offset -= length; + } + + if (length > (uint32_t)offset) { + addr = (char *)chunk->buf[i] + offset; + goto out; + } + offset -= length; + } + } + +out: + mutex_unlock(&table->mutex); + return addr; +} + int udma_init_hem_table(struct udma_dev *udma_dev, struct udma_hem_table *table, uint32_t type, uint64_t obj_size, uint64_t nobj) @@ -836,6 +905,644 @@ void udma_cleanup_hem_table(struct udma_dev *udma_dev, kfree(table->hem); } +static struct udma_hem_item * +hem_list_alloc_item(struct udma_dev *udma_dev, int start, int end, int count, + bool exist_bt) +{ + struct udma_hem_item *hem; + + hem = kzalloc(sizeof(*hem), GFP_KERNEL); + if (!hem) + return NULL; + + if (exist_bt) { + hem->addr = dma_alloc_coherent(udma_dev->dev, + count * BA_BYTE_LEN, + &hem->dma_addr, GFP_KERNEL); + if (!hem->addr) { + kfree(hem); + return NULL; + } + } + + hem->count = count; + hem->start = start; + hem->end = end; + INIT_LIST_HEAD(&hem->list); + INIT_LIST_HEAD(&hem->sibling); + + return hem; +} + +static void hem_list_free_item(struct udma_dev *udma_dev, + struct udma_hem_item *hem, bool exist_bt) +{ + if (exist_bt) + dma_free_coherent(udma_dev->dev, hem->count * BA_BYTE_LEN, + hem->addr, hem->dma_addr); + kfree(hem); +} + +static void hem_list_free_all(struct udma_dev *udma_dev, + struct list_head *head, bool exist_bt) +{ + struct udma_hem_item *hem, *temp_hem; + + list_for_each_entry_safe(hem, temp_hem, head, list) { + list_del(&hem->list); + hem_list_free_item(udma_dev, hem, exist_bt); + } +} + +static void hem_list_link_bt(struct udma_dev *udma_dev, void *base_addr, + uint64_t table_addr) +{ + *(uint64_t *)(base_addr) = table_addr; +} + +/* assign L0 table address to hem from root bt */ +static void hem_list_assign_bt(struct udma_dev *udma_dev, + struct udma_hem_item *hem, void *cpu_addr, + uint64_t phy_addr) +{ + hem->addr = cpu_addr; + hem->dma_addr = (dma_addr_t)phy_addr; +} + +static inline bool hem_list_page_is_in_range(struct udma_hem_item *hem, + int offset) +{ + return (hem->start <= offset && offset <= hem->end); +} + +static struct udma_hem_item *hem_list_search_item(struct list_head *ba_list, + int page_offset) +{ + struct udma_hem_item *found = NULL; + struct udma_hem_item *hem; + + list_for_each_entry(hem, ba_list, list) { + if (hem_list_page_is_in_range(hem, page_offset)) { + found = hem; + break; + } + } + + return found; +} + +static bool hem_list_is_bottom_bt(int hopnum, int bt_level) +{ + /* + * hopnum base address table levels + * 0 L0(buf) + * 1 L0 -> buf + * 2 L0 -> L1 -> buf + * 3 L0 -> L1 -> L2 -> buf + */ + return bt_level >= (hopnum ? hopnum - 1 : hopnum); +} + +/** + * calc base address entries num + * @hopnum: num of mutihop addressing + * @bt_level: base address table level + * @unit: ba entries per bt page + */ +static uint32_t hem_list_calc_ba_range(int hopnum, int bt_level, int unit) +{ + uint32_t step; + int max; + int i; + + if (hopnum <= bt_level) + return 0; + /* + * hopnum bt_level range + * 1 0 unit + * ------------ + * 2 0 unit * unit + * 2 1 unit + * ------------ + * 3 0 unit * unit * unit + * 3 1 unit * unit + * 3 2 unit + */ + step = 1; + max = hopnum - bt_level; + for (i = 0; i < max; i++) + step = step * unit; + + return step; +} + +/** + * calc the root ba entries which could cover all regions + * @regions: buf region array + * @region_cnt: array size of @regions + * @unit: ba entries per bt page + */ +static int udma_hem_list_calc_root_ba(const struct udma_buf_region *regions, + int region_cnt, int unit) +{ + struct udma_buf_region *r; + int total = 0; + int step; + int i; + + for (i = 0; i < region_cnt; i++) { + r = (struct udma_buf_region *)®ions[i]; + if (r->hopnum > 1) { + step = hem_list_calc_ba_range(r->hopnum, 1, unit); + if (step > 0) + total += (r->count + step - 1) / step; + } else { + total += r->count; + } + } + + return total; +} + +static int hem_list_alloc_mid_bt(struct udma_dev *udma_dev, + const struct udma_buf_region *r, int unit, + uint32_t offset, struct list_head *mid_bt, + struct list_head *btm_bt) +{ + struct udma_hem_item *hem_ptrs[UDMA_MAX_BT_LEVEL] = { NULL }; + struct list_head temp_list[UDMA_MAX_BT_LEVEL]; + struct udma_hem_item *cur, *pre; + const int hopnum = r->hopnum; + int start_aligned; + uint32_t step; + int distance; + int ret = 0; + int max_ofs; + int level; + int end; + + if (hopnum <= 1) + return 0; + + if (hopnum > UDMA_MAX_BT_LEVEL) { + dev_err(udma_dev->dev, "invalid hopnum %d!\n", hopnum); + return -EINVAL; + } + + if (offset < r->offset) { + dev_err(udma_dev->dev, "invalid offset %d, min %u!\n", + offset, r->offset); + return -EINVAL; + } + + distance = offset - r->offset; + max_ofs = r->offset + r->count - 1; + for (level = 0; level < hopnum; level++) + INIT_LIST_HEAD(&temp_list[level]); + + /* config L1 bt to last bt and link them to corresponding parent */ + for (level = 1; level < hopnum; level++) { + cur = hem_list_search_item(&mid_bt[level], offset); + if (cur) { + hem_ptrs[level] = cur; + continue; + } + + step = hem_list_calc_ba_range(hopnum, level, unit); + if (step < 1) { + ret = -EINVAL; + goto err_exit; + } + + start_aligned = (distance / step) * step + r->offset; + end = min_t(int, start_aligned + step - 1, max_ofs); + cur = hem_list_alloc_item(udma_dev, start_aligned, end, unit, + true); + if (!cur) { + ret = -ENOMEM; + goto err_exit; + } + hem_ptrs[level] = cur; + list_add(&cur->list, &temp_list[level]); + if (hem_list_is_bottom_bt(hopnum, level)) + list_add(&cur->sibling, &temp_list[0]); + + /* link bt to parent bt */ + if (level > 1) { + pre = hem_ptrs[level - 1]; + step = (cur->start - pre->start) / step * BA_BYTE_LEN; + hem_list_link_bt(udma_dev, (char *)pre->addr + step, + cur->dma_addr); + } + } + + list_splice(&temp_list[0], btm_bt); + for (level = 1; level < hopnum; level++) + list_splice(&temp_list[level], &mid_bt[level]); + + return 0; + +err_exit: + for (level = 1; level < hopnum; level++) + hem_list_free_all(udma_dev, &temp_list[level], true); + + return ret; +} + +static struct udma_hem_item * +alloc_root_hem(struct udma_dev *udma_dev, int unit, int *max_ba_num, + const struct udma_buf_region *regions, int region_cnt) +{ + const struct udma_buf_region *r; + struct udma_hem_item *hem; + int ba_num; + int offset; + + ba_num = udma_hem_list_calc_root_ba(regions, region_cnt, unit); + if (ba_num < 1) + return ERR_PTR(-ENOMEM); + + if (ba_num > unit) + return ERR_PTR(-ENOBUFS); + + offset = regions[0].offset; + /* indicate to last region */ + r = ®ions[region_cnt - 1]; + hem = hem_list_alloc_item(udma_dev, offset, r->offset + r->count - 1, + ba_num, true); + if (!hem) + return ERR_PTR(-ENOMEM); + + *max_ba_num = ba_num; + + return hem; +} + +static int alloc_fake_root_bt(struct udma_dev *udma_dev, void *cpu_base, + uint64_t phy_base, + const struct udma_buf_region *r, + struct list_head *branch_head, + struct list_head *leaf_head) +{ + struct udma_hem_item *hem; + + hem = hem_list_alloc_item(udma_dev, r->offset, r->offset + r->count - 1, + r->count, false); + if (!hem) + return -ENOMEM; + + hem_list_assign_bt(udma_dev, hem, cpu_base, phy_base); + list_add(&hem->list, branch_head); + list_add(&hem->sibling, leaf_head); + + return r->count; +} + +static int setup_middle_bt(struct udma_dev *udma_dev, void *cpu_base, + int unit, const struct udma_buf_region *r, + const struct list_head *branch_head) +{ + struct udma_hem_item *hem; + int total = 0; + int offset; + int step; + + step = hem_list_calc_ba_range(r->hopnum, 1, unit); + if (step < 1) + return -EINVAL; + + /* if exist mid bt, link L1 to L0 */ + list_for_each_entry(hem, branch_head, list) { + offset = (hem->start - r->offset) / step * BA_BYTE_LEN; + hem_list_link_bt(udma_dev, (char *)cpu_base + offset, + hem->dma_addr); + total++; + } + + return total; +} + +static int +setup_root_hem(struct udma_dev *udma_dev, struct udma_hem_list *hem_list, + int unit, int max_ba_num, struct udma_hem_head *head, + const struct udma_buf_region *regions, int region_cnt) +{ + const struct udma_buf_region *r; + struct udma_hem_item *root_hem; + uint64_t phy_base; + void *cpu_base; + int i, total; + int ret; + + root_hem = list_first_entry(&head->root, + struct udma_hem_item, list); + if (!root_hem) + return -ENOMEM; + + total = 0; + for (i = 0; i < region_cnt && total < max_ba_num; i++) { + r = ®ions[i]; + if (!r->count) + continue; + + /* all regions's mid[x][0] shared the root_bt's trunk */ + cpu_base = (char *)root_hem->addr + total * BA_BYTE_LEN; + phy_base = root_hem->dma_addr + total * BA_BYTE_LEN; + + /* if hopnum is 0 or 1, cut a new fake hem from the root bt + * which's address share to all regions. + */ + if (hem_list_is_bottom_bt(r->hopnum, 0)) + ret = alloc_fake_root_bt(udma_dev, cpu_base, phy_base, + r, &head->branch[i], + &head->leaf); + else + ret = setup_middle_bt(udma_dev, cpu_base, unit, r, + &hem_list->mid_bt[i][1]); + + if (ret < 0) + return ret; + + total += ret; + } + + list_splice(&head->leaf, &hem_list->btm_bt); + list_splice(&head->root, &hem_list->root_bt); + for (i = 0; i < region_cnt; i++) + list_splice(&head->branch[i], &hem_list->mid_bt[i][0]); + + return 0; +} + +static int hem_list_alloc_root_bt(struct udma_dev *udma_dev, + struct udma_hem_list *hem_list, int unit, + const struct udma_buf_region *regions, + int region_cnt) +{ + struct udma_hem_item *root_hem; + struct udma_hem_head head; + int max_ba_num; + int ret; + int i; + + /* Existed in hem list */ + root_hem = hem_list_search_item(&hem_list->root_bt, regions[0].offset); + if (root_hem) + return 0; + + max_ba_num = 0; + root_hem = alloc_root_hem(udma_dev, unit, &max_ba_num, regions, + region_cnt); + if (IS_ERR(root_hem)) + return PTR_ERR(root_hem); + + /* List head for storing all allocated HEM items */ + INIT_LIST_HEAD(&head.root); + INIT_LIST_HEAD(&head.leaf); + for (i = 0; i < region_cnt; i++) + INIT_LIST_HEAD(&head.branch[i]); + + hem_list->root_ba = root_hem->dma_addr; + list_add(&root_hem->list, &head.root); + ret = setup_root_hem(udma_dev, hem_list, unit, max_ba_num, &head, + regions, region_cnt); + if (ret) { + for (i = 0; i < region_cnt; i++) + hem_list_free_all(udma_dev, &head.branch[i], false); + + hem_list_free_all(udma_dev, &head.root, true); + } + + return ret; +} + +static void udma_hem_list_release(struct udma_dev *udma_dev, + struct udma_hem_list *hem_list) +{ + int i, j; + + for (i = 0; i < UDMA_MAX_BT_REGION; i++) + for (j = 0; j < UDMA_MAX_BT_LEVEL; j++) + hem_list_free_all(udma_dev, &hem_list->mid_bt[i][j], + j != 0); + + hem_list_free_all(udma_dev, &hem_list->root_bt, true); + INIT_LIST_HEAD(&hem_list->btm_bt); + hem_list->root_ba = 0; +} + +/* construct the base address table and link them by address hop config */ +static int udma_hem_list_request(struct udma_dev *udma_dev, + struct udma_hem_list *hem_list, + const struct udma_buf_region *regions, + int region_cnt, uint32_t bt_pg_shift) +{ + const struct udma_buf_region *r; + uint32_t ofs; + uint32_t end; + int unit; + int ret; + int i; + + if (region_cnt > UDMA_MAX_BT_REGION) { + dev_err(udma_dev->dev, "invalid region region_cnt %d!\n", + region_cnt); + return -EINVAL; + } + + unit = (1 << bt_pg_shift) / BA_BYTE_LEN; + for (i = 0; i < region_cnt; i++) { + r = ®ions[i]; + if (!r->count) + continue; + + end = r->offset + r->count; + for (ofs = r->offset; ofs < end; ofs += unit) { + ret = hem_list_alloc_mid_bt(udma_dev, r, unit, ofs, + hem_list->mid_bt[i], + &hem_list->btm_bt); + if (ret) { + dev_err(udma_dev->dev, + "alloc hem trunk fail ret = %d!\n", + ret); + goto err_alloc; + } + } + } + + ret = hem_list_alloc_root_bt(udma_dev, hem_list, unit, regions, + region_cnt); + if (ret) + dev_err(udma_dev->dev, "alloc hem root fail ret = %d!\n", ret); + else + return 0; + +err_alloc: + udma_hem_list_release(udma_dev, hem_list); + + return ret; +} + +static void udma_hem_list_init(struct udma_hem_list *hem_list) +{ + int i, j; + + INIT_LIST_HEAD(&hem_list->root_bt); + INIT_LIST_HEAD(&hem_list->btm_bt); + for (i = 0; i < UDMA_MAX_BT_REGION; i++) + for (j = 0; j < UDMA_MAX_BT_LEVEL; j++) + INIT_LIST_HEAD(&hem_list->mid_bt[i][j]); +} + +static void *udma_hem_list_find_mtt(struct udma_dev *udma_dev, + struct udma_hem_list *hem_list, int offset, + int *mtt_cnt, uint64_t *phy_addr) +{ + struct list_head *head = &hem_list->btm_bt; + struct udma_hem_item *hem; + int relative_offset = 0; + void *cpu_base = NULL; + uint64_t phy_base = 0; + + if (IS_ERR_OR_NULL(head->next)) + return cpu_base; + list_for_each_entry(hem, head, sibling) { + if (hem_list_page_is_in_range(hem, offset)) { + relative_offset = offset - hem->start; + cpu_base = (char *)hem->addr + relative_offset * + BA_BYTE_LEN; + phy_base = hem->dma_addr + relative_offset * + BA_BYTE_LEN; + relative_offset = hem->end + 1 - offset; + break; + } + } + + if (mtt_cnt) + *mtt_cnt = relative_offset; + + if (phy_addr) + *phy_addr = phy_base; + + return cpu_base; +} + +static inline bool mtr_has_mtt(struct udma_buf_attr *attr) +{ + uint32_t i; + + for (i = 0; i < attr->region_count; i++) + if (attr->region[i].hopnum != UDMA_HOP_NUM_0 && + attr->region[i].hopnum > 0) + return true; + + /* Mtr only has one root base address, when hopnum 0 means root base + * address equals the first buffer address. So all alloced memory must + * in a continuous space accessed by direct mode. + */ + return false; +} + +static inline size_t mtr_bufs_size(struct udma_buf_attr *attr) +{ + size_t size = 0; + uint32_t i; + + for (i = 0; i < attr->region_count; i++) + size += attr->region[i].size; + + return size; +} + +static bool need_split_huge_page(struct udma_hem_cfg *cfg) +{ + /* When HEM buffer uses 0-level addressing, the page size is + * equal to the whole buffer size. If the current MTR has multiple + * regions, we split the buffer into small pages(4k, required by + * UDMA). These pages will be used in multiple regions. + */ + return cfg->is_direct && cfg->region_count > 1; +} + +static int mtr_init_buf_cfg(struct udma_dev *udma_dev, + struct udma_buf_attr *attr, + struct udma_hem_cfg *cfg, + uint32_t *buf_page_shift, int unalinged_size) +{ + uint32_t page_count, region_count; + struct udma_buf_region *r; + int first_region_pad; + uint32_t page_shift; + size_t buf_size; + + /* If mtt is disabled, all pages must be within a continuous range */ + cfg->is_direct = !mtr_has_mtt(attr); + buf_size = mtr_bufs_size(attr); + if (need_split_huge_page(cfg)) { + /* When HEM buffer uses 0-level addressing, the page size is + * equal to the whole buffer size, and we split the buffer into + * small pages which is used to check whether the adjacent + * units are in the continuous space and its size is fixed to + * 4K based on hns ROCEE's requirement. + */ + page_shift = UDMA_HW_PAGE_SHIFT; + + cfg->buf_pg_count = 1; + cfg->buf_pg_shift = UDMA_HW_PAGE_SHIFT + + order_base_2(DIV_ROUND_UP(buf_size, UDMA_PAGE_SIZE)); + first_region_pad = 0; + } else { + page_shift = attr->page_shift; + cfg->buf_pg_count = DIV_ROUND_UP(buf_size + unalinged_size, + 1 << page_shift); + cfg->buf_pg_shift = page_shift; + first_region_pad = unalinged_size; + } + + /* Convert buffer size to page index and page count for each region. + * The buffer's offset needs to be appended to the first region. + */ + for (page_count = 0, region_count = 0; region_count < attr->region_count && + region_count < ARRAY_SIZE(cfg->region); region_count++) { + r = &cfg->region[region_count]; + r->offset = page_count; + buf_size = udma_hw_page_align(attr->region[region_count].size + + first_region_pad); + r->count = DIV_ROUND_UP(buf_size, 1 << page_shift); + first_region_pad = 0; + page_count += r->count; + r->hopnum = to_udma_hem_hopnum(attr->region[region_count].hopnum, + r->count); + } + + cfg->region_count = region_count; + *buf_page_shift = page_shift; + + return page_count; +} + +static int mtr_alloc_mtt(struct udma_dev *udma_dev, struct udma_mtr *mtr, + uint32_t ba_pg_shift) +{ + struct udma_hem_cfg *cfg = &mtr->hem_cfg; + int ret; + + udma_hem_list_init(&mtr->hem_list); + if (!cfg->is_direct) { + ret = udma_hem_list_request(udma_dev, &mtr->hem_list, + cfg->region, cfg->region_count, + ba_pg_shift); + if (ret) + return ret; + cfg->root_ba = mtr->hem_list.root_ba; + cfg->ba_pg_shift = ba_pg_shift; + } else { + cfg->ba_pg_shift = cfg->buf_pg_shift; + } + + return 0; +} + struct udma_buf *udma_buf_alloc(struct udma_dev *udma_dev, uint32_t size, uint32_t page_shift, uint32_t flags) { @@ -901,6 +1608,254 @@ struct udma_buf *udma_buf_alloc(struct udma_dev *udma_dev, uint32_t size, return buf; } + +static int mtr_alloc_bufs(struct udma_dev *udma_dev, struct udma_mtr *mtr, + struct udma_buf_attr *buf_attr, + uint64_t user_addr, bool is_user) +{ + struct ubcore_device *ubcore_dev = &udma_dev->ub_dev; + union ubcore_umem_flag access; + size_t total_size; + + total_size = mtr_bufs_size(buf_attr); + + if (is_user) { + mtr->kmem = NULL; + access.bs.non_pin = 0; + access.bs.writable = 1; + mtr->umem = ubcore_umem_get(ubcore_dev, user_addr, total_size, + access); + if (IS_ERR_OR_NULL(mtr->umem)) { + dev_err(udma_dev->dev, + "failed to get umem, ret = %ld.\n", + PTR_ERR(mtr->umem)); + return -ENOMEM; + } + } else { + mtr->umem = NULL; + mtr->kmem = udma_buf_alloc(udma_dev, total_size, + buf_attr->page_shift, + mtr->hem_cfg.is_direct ? + UDMA_BUF_DIRECT : 0); + if (IS_ERR(mtr->kmem)) { + dev_err(udma_dev->dev, + "failed to alloc kmem, ret = %ld.\n", + PTR_ERR(mtr->kmem)); + return PTR_ERR(mtr->kmem); + } + } + + return 0; +} + +int udma_get_umem_bufs(struct udma_dev *udma_dev, dma_addr_t *bufs, + int buf_cnt, struct ubcore_umem *umem, + uint32_t page_shift) +{ + struct scatterlist *sg; + int npage_per_sg; + dma_addr_t addr; + int npage = 0; + int total = 0; + uint32_t k; + int i; + + for_each_sg(umem->sg_head.sgl, sg, umem->sg_head.nents, k) { + npage_per_sg = sg_dma_len(sg) >> page_shift; + for (i = 0; i < npage_per_sg; i++) { + addr = sg_dma_address(sg) + (i << page_shift); + if (addr & ((1 << page_shift) - 1)) { + dev_err(udma_dev->dev, + "Umem addr not align to page_shift %d!\n", + page_shift); + return -EFAULT; + } + + bufs[total++] = addr; + if (total >= buf_cnt) + goto done; + + npage++; + } + } +done: + return total; +} + +static inline int mtr_check_direct_pages(dma_addr_t *pages, int page_cnt, + uint32_t page_shift) +{ + size_t page_sz = 1 << page_shift; + int i; + + for (i = 1; i < page_cnt; i++) + if (pages[i] - pages[i - 1] != page_sz) + return i; + + return 0; +} + +static int mtr_map_region(struct udma_dev *udma_dev, struct udma_mtr *mtr, + struct udma_buf_region *region, dma_addr_t *pages, + int max_count) +{ + int offset, end; + uint64_t *mtts; + uint64_t addr; + int npage = 0; + int count; + int i; + + offset = region->offset; + end = offset + region->count; + while (offset < end && npage < max_count) { + count = 0; + mtts = (uint64_t *)udma_hem_list_find_mtt(udma_dev, + &mtr->hem_list, + offset, &count, NULL); + if (!mtts) + return -ENOBUFS; + + for (i = 0; i < count && npage < max_count; i++) { + addr = pages[npage]; + mtts[i] = cpu_to_le64(addr); + npage++; + } + offset += count; + } + + return npage; +} + +int udma_mtr_map(struct udma_dev *udma_dev, struct udma_mtr *mtr, + dma_addr_t *pages, uint32_t page_count) +{ + struct device *dev = udma_dev->dev; + struct udma_buf_region *r; + uint32_t i, mapped_count; + int ret = 0; + + /* + * Only first page address was used as root ba when hopnum is 0, + * because the addresses of all pages are consecutive in this case. + */ + if (mtr->hem_cfg.is_direct) { + mtr->hem_cfg.root_ba = pages[0]; + return 0; + } + + for (i = 0, mapped_count = 0; i < mtr->hem_cfg.region_count && + mapped_count < page_count; i++) { + r = &mtr->hem_cfg.region[i]; + /* no need to map pages in this region when hopnum is 0 */ + if (!r->hopnum) { + mapped_count += r->count; + continue; + } + + if (r->offset + r->count > page_count) { + ret = -EINVAL; + dev_err(dev, + "failed to check mtr%u count %u + %u > %u.\n", + i, r->offset, r->count, page_count); + return ret; + } + + ret = mtr_map_region(udma_dev, mtr, r, &pages[r->offset], + page_count - mapped_count); + if (ret < 0) { + dev_err(dev, + "failed to map mtr%u offset %u, ret = %d.\n", + i, r->offset, ret); + return ret; + } + mapped_count += ret; + } + ret = 0; + + if (mapped_count < page_count) { + ret = -ENOBUFS; + dev_err(dev, "failed to map mtr pages count: %u < %u.\n", + mapped_count, page_count); + } + + return ret; +} + +int udma_get_kmem_bufs(struct udma_dev *udma_dev, dma_addr_t *bufs, + int buf_cnt, struct udma_buf *buf, + uint32_t page_shift) +{ + uint32_t offset, max_size; + int total = 0; + int i; + + if (page_shift > buf->trunk_shift) { + dev_err(udma_dev->dev, + "failed to check kmem buf shift %u > %u\n", + page_shift, buf->trunk_shift); + return -EINVAL; + } + + offset = 0; + max_size = buf->ntrunks << buf->trunk_shift; + for (i = 0; i < buf_cnt && offset < max_size; i++) { + bufs[total++] = udma_buf_dma_addr(buf, offset); + offset += (1 << page_shift); + } + + return total; +} + +static int mtr_map_bufs(struct udma_dev *udma_dev, struct udma_mtr *mtr, + int page_count, uint32_t page_shift) +{ + struct device *dev = udma_dev->dev; + dma_addr_t *pages; + int npage; + int ret; + + page_shift = need_split_huge_page(&mtr->hem_cfg) ? + UDMA_HW_PAGE_SHIFT : page_shift; + + /* alloc a tmp array to store buffer's dma address */ + pages = kvcalloc(page_count, sizeof(dma_addr_t), GFP_KERNEL); + if (!pages) + return -ENOMEM; + + if (mtr->umem) + npage = udma_get_umem_bufs(udma_dev, pages, page_count, + mtr->umem, page_shift); + else + npage = udma_get_kmem_bufs(udma_dev, pages, page_count, + mtr->kmem, page_shift); + if (npage != page_count) { + dev_err(dev, "failed to get mtr page %d != %d.\n", npage, + page_count); + ret = -ENOBUFS; + goto err_alloc_list; + } + + if (need_split_huge_page(&mtr->hem_cfg) && npage > 1) { + ret = mtr_check_direct_pages(pages, npage, page_shift); + if (ret) { + dev_err(dev, "failed to check %s page: %d / %d.\n", + mtr->umem ? "umtr" : "kmtr", ret, npage); + ret = -ENOBUFS; + goto err_alloc_list; + } + } + + ret = udma_mtr_map(udma_dev, mtr, pages, page_count); + if (ret) + dev_err(dev, "failed to map mtr pages, ret = %d.\n", ret); + +err_alloc_list: + kvfree(pages); + + return ret; +} + void udma_buf_free(struct udma_dev *udma_dev, struct udma_buf *buf) { struct udma_buf_list *trunks; @@ -921,3 +1876,156 @@ void udma_buf_free(struct udma_dev *udma_dev, struct udma_buf *buf) kfree(buf); } + +static void mtr_free_bufs(struct udma_dev *udma_dev, struct udma_mtr *mtr) +{ + /* release user buffers */ + ubcore_umem_release(mtr->umem); + mtr->umem = NULL; + + /* release kernel buffers */ + udma_buf_free(udma_dev, mtr->kmem); + mtr->kmem = NULL; +} + +static void mtr_free_mtt(struct udma_dev *udma_dev, struct udma_mtr *mtr) +{ + udma_hem_list_release(udma_dev, &mtr->hem_list); +} + +/* + * udma_mtr_create - Create memory translate region. + */ +int udma_mtr_create(struct udma_dev *udma_dev, struct udma_mtr *mtr, + struct udma_buf_attr *buf_attr, uint32_t ba_page_shift, + uint64_t user_addr, bool is_user) +{ + struct device *dev = udma_dev->dev; + uint32_t buf_page_shift = 0; + int buf_page_cnt; + int ret; + + buf_page_cnt = mtr_init_buf_cfg(udma_dev, buf_attr, &mtr->hem_cfg, + &buf_page_shift, + is_user ? user_addr & ~PAGE_MASK : 0); + if (buf_page_cnt < 1 || buf_page_shift < UDMA_HW_PAGE_SHIFT) { + dev_err(dev, "failed to init mtr cfg, count %d shift %u.\n", + buf_page_cnt, buf_page_shift); + return -EINVAL; + } + + ret = mtr_alloc_mtt(udma_dev, mtr, ba_page_shift); + if (ret) { + dev_err(dev, "failed to alloc mtr mtt, ret = %d.\n", ret); + return ret; + } + + /* The caller has its own buffer list and invokes the udma_mtr_map() + * to finish the MTT configuration. + */ + if (buf_attr->mtt_only) { + mtr->umem = NULL; + mtr->kmem = NULL; + return 0; + } + + ret = mtr_alloc_bufs(udma_dev, mtr, buf_attr, user_addr, is_user); + if (ret) { + dev_err(dev, "failed to alloc mtr bufs, ret = %d.\n", ret); + goto err_alloc_mtt; + } + + /* Write buffer's dma address to MTT */ + ret = mtr_map_bufs(udma_dev, mtr, buf_page_cnt, buf_page_shift); + if (ret) + dev_err(dev, "failed to map mtr bufs, ret = %d.\n", ret); + else + return 0; + + mtr_free_bufs(udma_dev, mtr); +err_alloc_mtt: + mtr_free_mtt(udma_dev, mtr); + return ret; +} + +void udma_mtr_destroy(struct udma_dev *udma_dev, struct udma_mtr *mtr) +{ + /* release multi-hop addressing resource */ + udma_hem_list_release(udma_dev, &mtr->hem_list); + + /* free buffers */ + mtr_free_bufs(udma_dev, mtr); +} + +int udma_mtr_find(struct udma_dev *udma_device, struct udma_mtr *mtr, + int offset, uint64_t *mtt_buf, int mtt_max, + uint64_t *base_addr) +{ + int mtt_count, left, start_idx, total; + struct udma_hem_cfg *cfg; + uint32_t npage; + uint64_t *mtts; + uint64_t addr; + + cfg = &mtr->hem_cfg; + total = 0; + + if (!mtt_buf || mtt_max < 1) + goto out; + + /* no mtt memory in direct mode, so just return the buffer address */ + if (cfg->is_direct) { + start_idx = offset >> UDMA_HW_PAGE_SHIFT; + for (mtt_count = 0; (uint32_t)mtt_count < cfg->region_count && + total < mtt_max; mtt_count++) { + npage = cfg->region[mtt_count].offset; + if ((int)npage < start_idx) + continue; + addr = cfg->root_ba + (npage << UDMA_HW_PAGE_SHIFT); + mtt_buf[total] = addr; + + total++; + } + goto out; + } + + start_idx = offset >> cfg->buf_pg_shift; + left = mtt_max; + while (left > 0) { + mtt_count = 0; + mtts = (uint64_t *)udma_hem_list_find_mtt(udma_device, + &mtr->hem_list, + start_idx + total, + &mtt_count, NULL); + if (!mtts || !mtt_count) + goto out; + + npage = min(mtt_count, left); + left -= npage; + for (mtt_count = 0; (uint32_t)mtt_count < npage; mtt_count++) + mtt_buf[total++] = le64_to_cpu(mtts[mtt_count]); + } + +out: + if (base_addr) + *base_addr = cfg->root_ba; + + return total; +} + +void udma_mtr_move(struct udma_mtr *from_mtr, struct udma_mtr *to_mtr) +{ + int i, j; + + *to_mtr = *from_mtr; + udma_hem_list_init(&to_mtr->hem_list); + + list_splice_init(&from_mtr->hem_list.root_bt, + &to_mtr->hem_list.root_bt); + list_splice_init(&from_mtr->hem_list.btm_bt, &to_mtr->hem_list.btm_bt); + + for (i = 0; i < UDMA_MAX_BT_REGION; i++) + for (j = 0; j < UDMA_MAX_BT_LEVEL; j++) + list_splice_init(&from_mtr->hem_list.mid_bt[i][j], + &to_mtr->hem_list.mid_bt[i][j]); +} diff --git a/drivers/ub/hw/hns3/hns3_udma_hem.h b/drivers/ub/hw/hns3/hns3_udma_hem.h index dbf8b594ede4..099050ab8167 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hem.h +++ b/drivers/ub/hw/hns3/hns3_udma_hem.h @@ -100,6 +100,23 @@ struct udma_hem_mhop { uint32_t l2_idx; /* level 2 base address table index */ }; +struct udma_hem_item { + struct list_head list; /* link all hems in the same bt level */ + struct list_head sibling; /* link all hems in last hop for mtt */ + void *addr; + dma_addr_t dma_addr; + size_t count; /* max ba numbers */ + int start; /* start buf offset in this hem */ + int end; /* end buf offset in this hem */ +}; + +/* All HEM itmes are linked in a tree structure */ +struct udma_hem_head { + struct list_head branch[UDMA_MAX_BT_REGION]; + struct list_head root; + struct list_head leaf; +}; + struct udma_buf *udma_buf_alloc(struct udma_dev *udma_dev, uint32_t size, uint32_t page_shift, uint32_t flags); void udma_buf_free(struct udma_dev *udma_dev, struct udma_buf *buf); -- Gitee From efe11e972e3c925f6f83643b6038c4c2b9d2c36a Mon Sep 17 00:00:00 2001 From: Weibo Zhao Date: Sat, 12 Aug 2023 16:59:07 +0800 Subject: [PATCH 05/42] hns3 udma: init software tables of qp/uar and others driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA ------------------------------------------------------ When driver insmoded, tables as qp/uar/jetty/jfs/jfr/jfc and others should be created. UAR(user-area-region) is an independent structure between context. It contains doorbell address and direct wqe address. Signed-off-by: Weibo Zhao --- drivers/ub/hw/hns3/Makefile | 2 +- drivers/ub/hw/hns3/hns3_udma_device.h | 25 +++ drivers/ub/hw/hns3/hns3_udma_hem.h | 2 + drivers/ub/hw/hns3/hns3_udma_hw.c | 11 ++ drivers/ub/hw/hns3/hns3_udma_jfc.h | 26 +++ drivers/ub/hw/hns3/hns3_udma_main.c | 258 ++++++++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_qp.c | 63 +++++++ 7 files changed, 386 insertions(+), 1 deletion(-) create mode 100644 drivers/ub/hw/hns3/hns3_udma_jfc.h create mode 100644 drivers/ub/hw/hns3/hns3_udma_qp.c diff --git a/drivers/ub/hw/hns3/Makefile b/drivers/ub/hw/hns3/Makefile index 122bb55bd26c..280b5406d1fe 100644 --- a/drivers/ub/hw/hns3/Makefile +++ b/drivers/ub/hw/hns3/Makefile @@ -8,6 +8,6 @@ ccflags-y += -I$(srctree)/drivers/net/ethernet/hisilicon/hns3/ \ -I$(srctree)/drivers/ub/urma/ $(MODULE_NAME)-objs := hns3_udma_hw.o hns3_udma_main.o hns3_udma_cmd.o \ - hns3_udma_hem.o + hns3_udma_hem.o hns3_udma_qp.o obj-$(CONFIG_UB_UDMA_HNS3) := hns3_udma.o diff --git a/drivers/ub/hw/hns3/hns3_udma_device.h b/drivers/ub/hw/hns3/hns3_udma_device.h index 7bc8b374a92b..c98cd83b39fa 100644 --- a/drivers/ub/hw/hns3/hns3_udma_device.h +++ b/drivers/ub/hw/hns3/hns3_udma_device.h @@ -31,6 +31,8 @@ #define PG_SHIFT_OFFSET (PAGE_SHIFT - 12) #define UDMA_MAX_IRQ_NUM 128 #define UDMA_QP_BANK_NUM 8 +#define QP_BANKID_SHIFT 3 +#define QP_BANKID_MASK GENMASK(2, 0) #define UDMA_QPC_SZ 512 #define UDMA_CQE_SZ 64 #define UDMA_SCCC_SZ 64 @@ -57,6 +59,7 @@ #define UDMA_MAX_BT_REGION 3 #define UDMA_MAX_BT_LEVEL 3 +#define UDMA_DEFAULT_MAX_JETTY_X_SHIFT 8 enum udma_reset_stage { UDMA_STATE_RST_DOWN = 2, @@ -452,6 +455,11 @@ struct udma_qp_table { struct udma_idx_table idx_table; }; +struct udma_eq_table { + uint32_t *idx_table; + struct udma_eq *eq; +}; + struct udma_jfc_table { struct xarray xa; struct udma_hem_table table; @@ -470,6 +478,11 @@ struct udma_jfr_table { struct udma_ida jfr_ida; }; +struct udma_jetty_table { + struct xarray xa; + struct udma_ida jetty_ida; +}; + struct udma_seg_table { struct udma_ida seg_ida; struct udma_hem_table table; @@ -484,6 +497,9 @@ struct udma_dev { bool dis_db; uint64_t reset_cnt; struct udma_netdev uboe; + + struct list_head pgdir_list; + struct mutex pgdir_mutex; uint8_t __iomem *reg_base; struct udma_caps caps; @@ -498,14 +514,21 @@ struct udma_dev { uint16_t func_id; uint32_t func_num; uint32_t cong_algo_tmpl_id; + struct udma_ida uar_ida; struct udma_jfs_table jfs_table; struct udma_jfr_table jfr_table; + struct udma_jetty_table jetty_table; struct udma_seg_table seg_table; struct udma_jfc_table jfc_table; struct udma_qp_table qp_table; + struct udma_eq_table eq_table; struct udma_hem_table qpc_timer_table; struct udma_hem_table cqc_timer_table; struct udma_hem_table gmv_table; + struct list_head qp_list; + spinlock_t qp_list_lock; + struct list_head dip_list; + spinlock_t dip_list_lock; }; static inline uint64_t to_hr_hw_page_addr(uint64_t addr) @@ -550,6 +573,8 @@ int udma_mtr_create(struct udma_dev *udma_dev, struct udma_mtr *mtr, struct udma_buf_attr *buf_attr, uint32_t ba_page_shift, uint64_t user_addr, bool is_user); void udma_mtr_destroy(struct udma_dev *udma_dev, struct udma_mtr *mtr); +int udma_init_qp_table(struct udma_dev *udma_dev); +void udma_cleanup_qp_table(struct udma_dev *udma_dev); void udma_cleanup_common_hem(struct udma_dev *udma_dev); int udma_init_common_hem(struct udma_dev *udma_dev); diff --git a/drivers/ub/hw/hns3/hns3_udma_hem.h b/drivers/ub/hw/hns3/hns3_udma_hem.h index 099050ab8167..2eacd3fc7799 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hem.h +++ b/drivers/ub/hw/hns3/hns3_udma_hem.h @@ -18,6 +18,8 @@ #include #include +#include "hns3_udma_device.h" + #define HEM_HOP_STEP_DIRECT 0xff #define HEM_INDEX_BUF BIT(0) #define HEM_INDEX_L0 BIT(1) diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.c b/drivers/ub/hw/hns3/hns3_udma_hw.c index 95418c09a8c0..0e971a2649ab 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hw.c +++ b/drivers/ub/hw/hns3/hns3_udma_hw.c @@ -162,6 +162,16 @@ static int udma_set_vf_switch_param(struct udma_dev *udma_dev) return 0; } +static void set_default_jetty_caps(struct udma_dev *dev) +{ + struct udma_caps *caps = &dev->caps; + + caps->num_jfc_shift = UDMA_DEFAULT_MAX_JETTY_X_SHIFT; + caps->num_jfs_shift = UDMA_DEFAULT_MAX_JETTY_X_SHIFT; + caps->num_jfr_shift = UDMA_DEFAULT_MAX_JETTY_X_SHIFT; + caps->num_jetty_shift = UDMA_DEFAULT_MAX_JETTY_X_SHIFT; +} + static int udma_query_caps(struct udma_dev *udma_dev) { enum udma_opcode_type opcode = UDMA_OPC_QUERY_PF_CAPS_NUM; @@ -337,6 +347,7 @@ static int udma_query_caps(struct udma_dev *udma_dev) QUERY_PF_CAPS_D_RQWQE_HOP_NUM_M, QUERY_PF_CAPS_D_RQWQE_HOP_NUM_S); + set_default_jetty_caps(udma_dev); return 0; } diff --git a/drivers/ub/hw/hns3/hns3_udma_jfc.h b/drivers/ub/hw/hns3/hns3_udma_jfc.h new file mode 100644 index 000000000000..dc4961830a31 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_jfc.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_JFC_H +#define _UDMA_JFC_H + +#include "hns3_udma_device.h" +static inline uint8_t get_jfc_bankid(uint64_t cqn) +{ + /* The lower 2 bits of CQN are used to hash to different banks */ + return (uint8_t)(cqn & GENMASK(1, 0)); +} + +#endif /* _UDMA_JFC_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c index 268094b1e25a..134a2b8def69 100644 --- a/drivers/ub/hw/hns3/hns3_udma_main.c +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -16,12 +16,253 @@ #include "urma/ubcore_api.h" #include "hns3_udma_device.h" #include "hns3_udma_hem.h" +#include "hns3_udma_jfc.h" #include "hns3_udma_cmd.h" static struct ubcore_ops g_udma_dev_ops = { }; +static void udma_cleanup_uar_table(struct udma_dev *dev) +{ + struct udma_ida *uar_ida = &dev->uar_ida; + + if (!ida_is_empty(&uar_ida->ida)) + dev_err(dev->dev, "IDA not empty in clean up uar table.\n"); + ida_destroy(&uar_ida->ida); +} + +static void udma_init_uar_table(struct udma_dev *udma_dev) +{ + struct udma_ida *uar_ida = &udma_dev->uar_ida; + + ida_init(&uar_ida->ida); + uar_ida->max = udma_dev->caps.num_uars - 1; + uar_ida->min = udma_dev->caps.reserved_uars; +} + +static void udma_cleanup_seg_table(struct udma_dev *dev) +{ + struct udma_ida *seg_ida = &dev->seg_table.seg_ida; + + if (!ida_is_empty(&seg_ida->ida)) + dev_err(dev->dev, "IDA not empty in clean up seg table.\n"); + ida_destroy(&seg_ida->ida); +} + +static void udma_init_seg_table(struct udma_dev *udma_dev) +{ + struct udma_ida *seg_ida = &udma_dev->seg_table.seg_ida; + + ida_init(&seg_ida->ida); + seg_ida->max = udma_dev->caps.num_mtpts - 1; + seg_ida->min = udma_dev->caps.reserved_mrws; +} + +static void udma_cleanup_jfc_table(struct udma_dev *udma_dev) +{ + struct udma_jfc_table *jfc_table = &udma_dev->jfc_table; + struct udma_jfc *jfc; + unsigned long index; + int i; + + for (i = 0; i < UDMA_CQ_BANK_NUM; i++) { + if (!ida_is_empty(&jfc_table->bank[i].ida)) + dev_err(udma_dev->dev, + "IDA not empty in clean up jfc bank[%d] table\n", + i); + ida_destroy(&jfc_table->bank[i].ida); + } + + if (!xa_empty(&jfc_table->xa)) { + dev_err(udma_dev->dev, "JFC not empty\n"); + xa_for_each(&jfc_table->xa, index, jfc) + udma_table_put(udma_dev, &jfc_table->table, index); + } + xa_destroy(&jfc_table->xa); +} + +static void udma_init_jfc_table(struct udma_dev *udma_dev) +{ + struct udma_jfc_table *jfc_table = &udma_dev->jfc_table; + uint32_t reserved_from_bot; + uint32_t i; + + mutex_init(&jfc_table->bank_mutex); + xa_init(&jfc_table->xa); + + reserved_from_bot = 1; + + for (i = 0; i < reserved_from_bot; i++) { + jfc_table->bank[get_jfc_bankid(i)].inuse++; + jfc_table->bank[get_jfc_bankid(i)].min++; + } + + for (i = 0; i < UDMA_CQ_BANK_NUM; i++) { + ida_init(&jfc_table->bank[i].ida); + jfc_table->bank[i].max = (1 << udma_dev->caps.num_jfc_shift) / + UDMA_CQ_BANK_NUM - 1; + } +} + +static void udma_cleanup_jfr_table(struct udma_dev *dev) +{ + struct udma_jfr_table *jfr_table = &dev->jfr_table; + struct udma_ida *jfr_ida = &jfr_table->jfr_ida; + unsigned long index = 0; + struct udma_jfr *jfr; + + if (!ida_is_empty(&jfr_ida->ida)) + dev_err(dev->dev, "IDA not empty in clean up jfr table.\n"); + ida_destroy(&jfr_ida->ida); + + if (!xa_empty(&jfr_table->xa)) { + dev_err(dev->dev, "JFR not empty\n"); + xa_for_each(&jfr_table->xa, index, jfr) + udma_table_put(dev, &jfr_table->table, index); + } + xa_destroy(&jfr_table->xa); +} + +static void udma_init_jfr_table(struct udma_dev *dev) +{ + struct udma_jfr_table *jfr_table = &dev->jfr_table; + struct udma_ida *jfr_ida = &jfr_table->jfr_ida; + + xa_init(&jfr_table->xa); + ida_init(&jfr_ida->ida); + jfr_ida->max = (1 << dev->caps.num_jfr_shift) - 1; + /* reserve jfr id 0 */ + jfr_ida->min = 1; +} + +static void udma_cleanup_jfs_table(struct udma_dev *dev) +{ + struct udma_jfs_table *jfs_table = &dev->jfs_table; + struct udma_ida *jfs_ida = &jfs_table->jfs_ida; + + if (!ida_is_empty(&jfs_ida->ida)) + dev_err(dev->dev, "IDA not empty in clean up jfs table.\n"); + ida_destroy(&jfs_ida->ida); + + if (!xa_empty(&jfs_table->xa)) + dev_err(dev->dev, "JFS table not empty.\n"); + xa_destroy(&jfs_table->xa); +} + +static void udma_init_jfs_table(struct udma_dev *dev) +{ + struct udma_jfs_table *jfs_table = &dev->jfs_table; + struct udma_ida *jfs_ida = &jfs_table->jfs_ida; + + xa_init(&jfs_table->xa); + ida_init(&jfs_ida->ida); + jfs_ida->max = (1 << dev->caps.num_jfs_shift) - 1; + /* reserve jfs id 0 */ + jfs_ida->min = 1; +} + +static void udma_cleanup_jetty_table(struct udma_dev *dev) +{ + struct udma_jetty_table *jetty_table = &dev->jetty_table; + struct udma_ida *jetty_ida = &jetty_table->jetty_ida; + + if (!ida_is_empty(&jetty_ida->ida)) + dev_err(dev->dev, "IDA not empty in clean up jetty table.\n"); + ida_destroy(&jetty_ida->ida); + + if (!xa_empty(&jetty_table->xa)) + dev_err(dev->dev, "Jetty table not empty.\n"); + xa_destroy(&jetty_table->xa); +} + +static void udma_init_jetty_table(struct udma_dev *dev) +{ + struct udma_jetty_table *jetty_table = &dev->jetty_table; + struct udma_ida *jetty_ida = &jetty_table->jetty_ida; + + xa_init(&jetty_table->xa); + ida_init(&jetty_ida->ida); + jetty_ida->max = (1 << dev->caps.num_jetty_shift) - 1; + /* reserve jetty id 0 */ + jetty_ida->min = 1; +} + +int udma_init_eq_idx_table(struct udma_dev *udma_dev) +{ + uint32_t eq_num; + + eq_num = udma_dev->caps.num_comp_vectors + + udma_dev->caps.num_aeq_vectors; + udma_dev->eq_table.idx_table = kcalloc(eq_num, sizeof(uint32_t), + GFP_KERNEL); + if (ZERO_OR_NULL_PTR(udma_dev->eq_table.idx_table)) + return -ENOMEM; + + return 0; +} + +int udma_setup_hca(struct udma_dev *udma_dev) +{ + struct device *dev = udma_dev->dev; + int ret; + + INIT_LIST_HEAD(&udma_dev->qp_list); + spin_lock_init(&udma_dev->qp_list_lock); + INIT_LIST_HEAD(&udma_dev->dip_list); + spin_lock_init(&udma_dev->dip_list_lock); + + if (udma_dev->caps.flags & UDMA_CAP_FLAG_CQ_RECORD_DB || + udma_dev->caps.flags & UDMA_CAP_FLAG_QP_RECORD_DB) { + INIT_LIST_HEAD(&udma_dev->pgdir_list); + mutex_init(&udma_dev->pgdir_mutex); + } + + udma_init_uar_table(udma_dev); + + ret = udma_init_qp_table(udma_dev); + if (ret) { + dev_err(dev, "Failed to init qp_table.\n"); + goto err_uar_table_free; + } + + udma_init_seg_table(udma_dev); + udma_init_jfc_table(udma_dev); + udma_init_jfr_table(udma_dev); + udma_init_jfs_table(udma_dev); + udma_init_jetty_table(udma_dev); + ret = udma_init_eq_idx_table(udma_dev); + if (ret) { + dev_err(dev, "Failed to init eq_table.\n"); + goto err_eq_table; + } + + return 0; +err_eq_table: + udma_cleanup_jetty_table(udma_dev); + udma_cleanup_jfs_table(udma_dev); + udma_cleanup_jfr_table(udma_dev); + udma_cleanup_jfc_table(udma_dev); + udma_cleanup_seg_table(udma_dev); + udma_cleanup_qp_table(udma_dev); + +err_uar_table_free: + udma_cleanup_uar_table(udma_dev); + return ret; +} + +void udma_teardown_hca(struct udma_dev *udma_dev) +{ + kfree(udma_dev->eq_table.idx_table); + udma_cleanup_jetty_table(udma_dev); + udma_cleanup_jfs_table(udma_dev); + udma_cleanup_jfr_table(udma_dev); + udma_cleanup_jfc_table(udma_dev); + udma_cleanup_seg_table(udma_dev); + udma_cleanup_qp_table(udma_dev); + udma_cleanup_uar_table(udma_dev); +} + int udma_init_common_hem(struct udma_dev *udma_dev) { struct device *dev = udma_dev->dev; @@ -285,6 +526,11 @@ int udma_hnae_client_init(struct udma_dev *udma_dev) goto error_failed_hem_init; } + ret = udma_setup_hca(udma_dev); + if (ret) { + dev_err(dev, "setup hca failed!\n"); + goto error_failed_setup; + } ret = udma_dev->hw->hw_init(udma_dev); if (ret) { dev_err(dev, "hw_init failed!\n"); @@ -303,6 +549,9 @@ int udma_hnae_client_init(struct udma_dev *udma_dev) udma_dev->hw->hw_exit(udma_dev); error_failed_engine_init: + udma_teardown_hca(udma_dev); + +error_failed_setup: udma_cleanup_hem(udma_dev); error_failed_hem_init: @@ -325,6 +574,15 @@ void udma_hnae_client_exit(struct udma_dev *udma_dev) if (udma_dev->hw->hw_exit) udma_dev->hw->hw_exit(udma_dev); + + udma_teardown_hca(udma_dev); + + udma_cleanup_hem(udma_dev); + + if (udma_dev->cmd_mod) + udma_cmd_use_polling(udma_dev); + + udma_cmd_cleanup(udma_dev); if (udma_dev->hw->cmq_exit) udma_dev->hw->cmq_exit(udma_dev); } diff --git a/drivers/ub/hw/hns3/hns3_udma_qp.c b/drivers/ub/hw/hns3/hns3_udma_qp.c new file mode 100644 index 000000000000..609d93b2a085 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_qp.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include +#include "hns3_udma_jfc.h" +#include "hns3_udma_hem.h" +static inline uint8_t get_qp_bankid(uint64_t qpn) +{ + /* The lower 3 bits of QPN are used to hash to different banks */ + return (uint8_t)(qpn & QP_BANKID_MASK); +} + +int udma_init_qp_table(struct udma_dev *dev) +{ + struct udma_qp_table *qp_table = &dev->qp_table; + uint32_t reserved_from_bot; + uint32_t i; + + qp_table->idx_table.spare_idx = kcalloc(dev->caps.num_qps, + sizeof(uint32_t), GFP_KERNEL); + if (!qp_table->idx_table.spare_idx) + return -ENOMEM; + + mutex_init(&qp_table->bank_mutex); + xa_init(&qp_table->xa); + + reserved_from_bot = dev->caps.reserved_qps; + + for (i = 0; i < reserved_from_bot; i++) { + dev->qp_table.bank[get_qp_bankid(i)].inuse++; + dev->qp_table.bank[get_qp_bankid(i)].min++; + } + + for (i = 0; i < UDMA_QP_BANK_NUM; i++) { + ida_init(&dev->qp_table.bank[i].ida); + dev->qp_table.bank[i].max = dev->caps.num_qps / + UDMA_QP_BANK_NUM - 1; + dev->qp_table.bank[i].next = dev->qp_table.bank[i].min; + } + + return 0; +} + +void udma_cleanup_qp_table(struct udma_dev *dev) +{ + int i; + + for (i = 0; i < UDMA_QP_BANK_NUM; i++) + ida_destroy(&dev->qp_table.bank[i].ida); + kfree(dev->qp_table.idx_table.spare_idx); +} -- Gitee From a214398c5e6f765062a0060f56bed355573cf80a Mon Sep 17 00:00:00 2001 From: Weibo Zhao Date: Tue, 15 Aug 2023 15:16:34 +0800 Subject: [PATCH 06/42] hns3 udma: add feature for eq driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA ---------------------------------------------------- JFCE (Jetty for complete event) and JFAE (jetty for async event) events are two kinds of event elements of the EQ(event queue) which are CEQ and AEQ. Event triggered by interrupt. Signed-off-by: Weibo Zhao --- drivers/ub/hw/hns3/Makefile | 2 +- drivers/ub/hw/hns3/hns3_udma_common.h | 4 + drivers/ub/hw/hns3/hns3_udma_device.h | 109 ++++ drivers/ub/hw/hns3/hns3_udma_eq.c | 738 ++++++++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_eq.h | 104 ++++ drivers/ub/hw/hns3/hns3_udma_hem.h | 13 + drivers/ub/hw/hns3/hns3_udma_hw.c | 4 + drivers/ub/hw/hns3/hns3_udma_hw.h | 10 + drivers/ub/hw/hns3/hns3_udma_main.c | 11 + drivers/ub/hw/hns3/hns3_udma_qp.c | 24 + drivers/ub/hw/hns3/hns3_udma_qp.h | 34 ++ 11 files changed, 1052 insertions(+), 1 deletion(-) create mode 100644 drivers/ub/hw/hns3/hns3_udma_eq.c create mode 100644 drivers/ub/hw/hns3/hns3_udma_eq.h create mode 100644 drivers/ub/hw/hns3/hns3_udma_qp.h diff --git a/drivers/ub/hw/hns3/Makefile b/drivers/ub/hw/hns3/Makefile index 280b5406d1fe..2cae99de3143 100644 --- a/drivers/ub/hw/hns3/Makefile +++ b/drivers/ub/hw/hns3/Makefile @@ -8,6 +8,6 @@ ccflags-y += -I$(srctree)/drivers/net/ethernet/hisilicon/hns3/ \ -I$(srctree)/drivers/ub/urma/ $(MODULE_NAME)-objs := hns3_udma_hw.o hns3_udma_main.o hns3_udma_cmd.o \ - hns3_udma_hem.o hns3_udma_qp.o + hns3_udma_hem.o hns3_udma_qp.o hns3_udma_eq.o obj-$(CONFIG_UB_UDMA_HNS3) := hns3_udma.o diff --git a/drivers/ub/hw/hns3/hns3_udma_common.h b/drivers/ub/hw/hns3/hns3_udma_common.h index ec6bc584c640..a092f66e7f33 100644 --- a/drivers/ub/hw/hns3/hns3_udma_common.h +++ b/drivers/ub/hw/hns3/hns3_udma_common.h @@ -24,6 +24,10 @@ #define udma_get_field(origin, mask, shift) \ ((le32_to_cpu(origin) & (mask)) >> (uint32_t)(shift)) +#define udma_get_field64(origin, mask, shift) \ + ((le64_to_cpu(origin) & (mask)) >> (uint32_t)(shift)) +#define udma_get_bit(origin, shift) \ + udma_get_field((origin), (1ul << (shift)), (shift)) #define udma_set_field(origin, mask, shift, val) \ do { \ diff --git a/drivers/ub/hw/hns3/hns3_udma_device.h b/drivers/ub/hw/hns3/hns3_udma_device.h index c98cd83b39fa..76074f264955 100644 --- a/drivers/ub/hw/hns3/hns3_udma_device.h +++ b/drivers/ub/hw/hns3/hns3_udma_device.h @@ -30,6 +30,9 @@ /* Configure to HW for PAGE_SIZE larger than 4KB */ #define PG_SHIFT_OFFSET (PAGE_SHIFT - 12) #define UDMA_MAX_IRQ_NUM 128 + +#define MTT_MIN_COUNT 2 + #define UDMA_QP_BANK_NUM 8 #define QP_BANKID_SHIFT 3 #define QP_BANKID_MASK GENMASK(2, 0) @@ -59,8 +62,26 @@ #define UDMA_MAX_BT_REGION 3 #define UDMA_MAX_BT_LEVEL 3 +#define EQ_ENABLE 1 +#define EQ_DISABLE 0 +#define UDMA_CEQ 0 +#define UDMA_AEQ 1 +#define UDMA_AEQ_DEFAULT_BURST_NUM 0x0 +#define UDMA_AEQ_DEFAULT_INTERVAL 0x0 +#define UDMA_CEQ_DEFAULT_BURST_NUM 0x0 +#define UDMA_CEQ_DEFAULT_INTERVAL 0x0 +#define UDMA_VF_EQ_DB_CFG0_REG 0x238 +#define UDMA_VF_ABN_INT_CFG_REG 0x13000 +#define UDMA_VF_ABN_INT_ST_REG 0x13004 +#define UDMA_VF_ABN_INT_EN_REG 0x13008 +#define UDMA_VF_EVENT_INT_EN_REG 0x1300c +#define EQ_REG_OFFSET 0x4 #define UDMA_DEFAULT_MAX_JETTY_X_SHIFT 8 +enum { + NO_ARMED = 0x0 +}; + enum udma_reset_stage { UDMA_STATE_RST_DOWN = 2, UDMA_STATE_RST_UNINIT, @@ -75,6 +96,18 @@ enum udma_instance_state { UDMA_STATE_UNINIT, }; +enum udma_event { + UDMA_EVENT_TYPE_COMM_EST = 0x03, + UDMA_EVENT_TYPE_WQ_CATAS_ERROR = 0x05, + UDMA_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR = 0x06, + UDMA_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR = 0x07, + UDMA_EVENT_TYPE_JFR_LIMIT_REACH = 0x08, + UDMA_EVENT_TYPE_JFR_LAST_WQE_REACH = 0x09, + UDMA_EVENT_TYPE_JFC_ACCESS_ERROR = 0x0b, + UDMA_EVENT_TYPE_JFC_OVERFLOW = 0x0c, + UDMA_EVENT_TYPE_MB = 0x13, +}; + enum { /* discard BIT(2), reserved for compatibility */ @@ -172,6 +205,41 @@ struct udma_mtr { struct udma_hem_cfg hem_cfg; /* config for hardware addressing */ }; +struct udma_ceqe { + uint32_t comp; + uint32_t rsv[15]; +}; + +struct udma_aeqe { + uint32_t asyn; + union { + struct { + uint32_t num; + uint32_t rsv0; + uint32_t rsv1; + } queue_event; + + struct { + uint64_t out_param; + uint16_t token; + uint8_t status; + uint8_t rsv0; + } __packed cmd; + } event; + uint32_t rsv[12]; +}; + +struct udma_work { + struct udma_dev *udma_dev; + struct work_struct work; + int event_type; + int sub_type; + struct udma_aeqe aeqe; + uint32_t queue_num; + uint32_t eq_ci; + int eqn; +}; + struct udma_dev; struct udma_cmd_context { struct completion done; @@ -280,6 +348,8 @@ struct udma_hw { int (*clear_hem)(struct udma_dev *udma_dev, struct udma_hem_table *table, int obj, int step_idx); + int (*init_eq)(struct udma_dev *udma_dev); + void (*cleanup_eq)(struct udma_dev *udma_dev); }; struct udma_caps { @@ -444,6 +514,30 @@ struct udma_bank { uint32_t next; /* Next ID to allocate. */ }; +#define TRACE_AEQE_LEN_MAX 64 + +struct udma_eq { + struct udma_dev *udma_dev; + void __iomem *db_reg; + + int type_flag; /* Aeq:1 ceq:0 */ + int eqn; + uint32_t entries; + int eqe_size; + int irq; + uint32_t cons_index; + int over_ignore; + int coalesce; + int arm_st; + int hop_num; + struct udma_mtr mtr; + uint16_t eq_max_cnt; + uint32_t eq_period; + int shift; + int event_type; + int sub_type; +}; + struct udma_qp_table { struct xarray xa; struct udma_hem_table qp_table; @@ -505,12 +599,14 @@ struct udma_dev { int irq_num; int irq[UDMA_MAX_IRQ_NUM]; + const char *irq_names[UDMA_MAX_IRQ_NUM]; char dev_name[UBCORE_MAX_DEV_NAME]; uint64_t sys_image_guid; struct udma_cmdq cmd; int cmd_mod; const struct udma_hw *hw; void *priv; + struct workqueue_struct *irq_workq; uint16_t func_id; uint32_t func_num; uint32_t cong_algo_tmpl_id; @@ -531,6 +627,13 @@ struct udma_dev { spinlock_t dip_list_lock; }; +static inline void *udma_buf_offset(struct udma_buf *buf, + uint32_t offset) +{ + return (char *)(buf->trunk_list[offset >> buf->trunk_shift].buf) + + (offset & ((1 << buf->trunk_shift) - 1)); +} + static inline uint64_t to_hr_hw_page_addr(uint64_t addr) { return addr >> UDMA_HW_PAGE_SHIFT; @@ -549,6 +652,11 @@ static inline uint32_t to_udma_hem_hopnum(uint32_t hopnum, uint32_t count) return 0; } +static inline uint32_t to_udma_hw_page_shift(uint32_t page_shift) +{ + return page_shift - UDMA_HW_PAGE_SHIFT; +} + static inline dma_addr_t udma_buf_dma_addr(struct udma_buf *buf, uint32_t offset) { @@ -577,5 +685,6 @@ int udma_init_qp_table(struct udma_dev *udma_dev); void udma_cleanup_qp_table(struct udma_dev *udma_dev); void udma_cleanup_common_hem(struct udma_dev *udma_dev); int udma_init_common_hem(struct udma_dev *udma_dev); +void udma_destroy_eqc(struct udma_dev *udma_dev, int eqn, uint32_t eq_cmd); #endif /* _UDMA_DEVICE_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_eq.c b/drivers/ub/hw/hns3/hns3_udma_eq.c new file mode 100644 index 000000000000..686be0cc8c55 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_eq.c @@ -0,0 +1,738 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include +#include "hnae3.h" +#include "hns3_udma_hem.h" +#include "hns3_udma_eq.h" +#include "hns3_udma_qp.h" + +static int alloc_eq_buf(struct udma_dev *udma_dev, struct udma_eq *eq) +{ + struct udma_buf_attr buf_attr = {}; + int err; + + if (udma_dev->caps.eqe_hop_num == UDMA_HOP_NUM_0) + eq->hop_num = 0; + else + eq->hop_num = udma_dev->caps.eqe_hop_num; + + buf_attr.page_shift = udma_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT; + buf_attr.region[0].size = eq->entries * eq->eqe_size; + buf_attr.region[0].hopnum = eq->hop_num; + buf_attr.region_count = 1; + + err = udma_mtr_create(udma_dev, &eq->mtr, &buf_attr, + udma_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT, 0, + 0); + if (err) + dev_err(udma_dev->dev, + "Failed to alloc EQE mtr, err %d\n", err); + + return err; +} + +static void init_eq_config(struct udma_dev *udma_dev, struct udma_eq *eq) +{ + eq->db_reg = udma_dev->reg_base + UDMA_VF_EQ_DB_CFG0_REG; + eq->cons_index = 0; + eq->over_ignore = UDMA_EQ_OVER_IGNORE_0; + eq->coalesce = UDMA_EQ_COALESCE_0; + eq->arm_st = UDMA_EQ_ALWAYS_ARMED; + eq->shift = ilog2((uint32_t)eq->entries); +} + +static int config_eqc(struct udma_dev *udma_dev, struct udma_eq *eq, + void *mb_buf) +{ + uint64_t eqe_ba[MTT_MIN_COUNT] = {}; + struct udma_eq_context *eqc; + uint64_t bt_ba = 0; + int count; + + eqc = (struct udma_eq_context *)mb_buf; + memset(eqc, 0, sizeof(struct udma_eq_context)); + + /* if not multi-hop, eqe buffer only use one trunk */ + count = udma_mtr_find(udma_dev, &eq->mtr, 0, eqe_ba, MTT_MIN_COUNT, + &bt_ba); + if (count < 1) { + dev_err(udma_dev->dev, "failed to find EQE mtr\n"); + return -ENOBUFS; + } + + udma_reg_write(eqc, EQC_EQ_ST, UDMA_EQ_STATE_VALID); + udma_reg_write(eqc, EQC_EQE_HOP_NUM, eq->hop_num); + udma_reg_write(eqc, EQC_OVER_IGNORE, eq->over_ignore); + udma_reg_write(eqc, EQC_COALESCE, eq->coalesce); + udma_reg_write(eqc, EQC_ARM_ST, eq->arm_st); + udma_reg_write(eqc, EQC_EQN, eq->eqn); + udma_reg_write(eqc, EQC_EQE_CNT, UDMA_EQ_INIT_EQE_CNT); + udma_reg_write(eqc, EQC_EQE_BA_PG_SZ, + to_udma_hw_page_shift(eq->mtr.hem_cfg.ba_pg_shift)); + udma_reg_write(eqc, EQC_EQE_BUF_PG_SZ, + to_udma_hw_page_shift(eq->mtr.hem_cfg.buf_pg_shift)); + udma_reg_write(eqc, EQC_EQ_PROD_INDX, UDMA_EQ_INIT_PROD_IDX); + udma_reg_write(eqc, EQC_EQ_MAX_CNT, eq->eq_max_cnt); + + udma_reg_write(eqc, EQC_EQ_PERIOD, eq->eq_period); + udma_reg_write(eqc, EQC_EQE_REPORT_TIMER, UDMA_EQ_INIT_REPORT_TIMER); + udma_reg_write(eqc, EQC_EQE_BA_L, bt_ba >> EQC_EQE_BA_L_SHIFT); + udma_reg_write(eqc, EQC_EQE_BA_H, bt_ba >> EQC_EQE_BA_H_SHIFT); + udma_reg_write(eqc, EQC_SHIFT, eq->shift); + udma_reg_write(eqc, EQC_MSI_INDX, UDMA_EQ_INIT_MSI_IDX); + udma_reg_write(eqc, EQC_CUR_EQE_BA_L, eqe_ba[0] >> + EQC_CUR_EQE_BA_L_SHIFT); + udma_reg_write(eqc, EQC_CUR_EQE_BA_M, eqe_ba[0] >> + EQC_CUR_EQE_BA_M_SHIFT); + udma_reg_write(eqc, EQC_CUR_EQE_BA_H, eqe_ba[0] >> + EQC_CUR_EQE_BA_H_SHIFT); + udma_reg_write(eqc, EQC_EQ_CONS_INDX, UDMA_EQ_INIT_CONS_IDX); + udma_reg_write(eqc, EQC_NEX_EQE_BA_L, eqe_ba[1] >> + EQC_NEX_EQE_BA_L_SHIFT); + udma_reg_write(eqc, EQC_NEX_EQE_BA_H, eqe_ba[1] >> + EQC_NEX_EQE_BA_H_SHIFT); + udma_reg_write(eqc, EQC_EQE_SIZE, eq->eqe_size == UDMA_EQE_SIZE); + + return 0; +} + +static void free_eq_buf(struct udma_dev *udma_dev, struct udma_eq *eq) +{ + udma_mtr_destroy(udma_dev, &eq->mtr); +} + +static int udma_create_eq(struct udma_dev *udma_dev, struct udma_eq *eq, + uint32_t eq_cmd) +{ + struct udma_cmd_mailbox *mailbox; + struct udma_cmq_desc desc; + struct udma_mbox *mb; + int ret; + + /* Allocate mailbox memory */ + mailbox = udma_alloc_cmd_mailbox(udma_dev); + if (IS_ERR_OR_NULL(mailbox)) + return -ENOMEM; + + ret = config_eqc(udma_dev, eq, mailbox->buf); + if (ret) + goto err_cmd_mbox; + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mbox_desc_init(mb, mailbox->dma, 0, eq->eqn, eq_cmd); + ret = udma_cmd_mbox(udma_dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); + if (ret) + dev_err(udma_dev->dev, "[mailbox cmd] create eqc failed.\n"); + +err_cmd_mbox: + udma_free_cmd_mailbox(udma_dev, mailbox); + + return ret; +} + +static struct udma_aeqe *next_aeqe_sw_v2(struct udma_eq *eq) +{ + struct udma_aeqe *aeqe; + + aeqe = (struct udma_aeqe *)udma_buf_offset(eq->mtr.kmem, + (eq->cons_index & (eq->entries - 1)) * + eq->eqe_size); + + return (udma_get_bit(aeqe->asyn, UDMA_AEQ_AEQE_OWNER_S) ^ + !!(eq->cons_index & eq->entries)) ? aeqe : NULL; +} + +static void aeq_event_dump(struct device *dev, struct udma_work *irq_work) +{ + switch (irq_work->event_type) { + case UDMA_EVENT_TYPE_COMM_EST: + break; + case UDMA_EVENT_TYPE_WQ_CATAS_ERROR: + dev_err(dev, "Local work queue 0x%x catast error, sub_event type is: 0x%x\n", + irq_work->queue_num, irq_work->sub_type); + break; + case UDMA_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: + dev_err(dev, + "Invalid request local work queue 0x%x error.\n", + irq_work->queue_num); + break; + case UDMA_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: + dev_err(dev, + "Local access violation work queue 0x%x error, sub_event type is: 0x%x\n", + irq_work->queue_num, irq_work->sub_type); + break; + case UDMA_EVENT_TYPE_JFR_LIMIT_REACH: + dev_warn(dev, "JFR limit reach.\n"); + break; + case UDMA_EVENT_TYPE_JFR_LAST_WQE_REACH: + dev_warn(dev, "JFR last wqe reach.\n"); + break; + case UDMA_EVENT_TYPE_JFC_ACCESS_ERROR: + dev_err(dev, "JFC 0x%x access err.\n", + irq_work->queue_num); + break; + case UDMA_EVENT_TYPE_JFC_OVERFLOW: + dev_warn(dev, "JFC 0x%x overflow\n", + irq_work->queue_num); + break; + default: + break; + } +} + +static void aeq_event_report(struct udma_dev *udma_dev, + struct udma_work *irq_work) +{ + uint32_t queue_num = irq_work->queue_num; + struct udma_aeqe *aeqe = &irq_work->aeqe; + int event_type = irq_work->event_type; + + switch (event_type) { + case UDMA_EVENT_TYPE_COMM_EST: + case UDMA_EVENT_TYPE_WQ_CATAS_ERROR: + case UDMA_EVENT_TYPE_JFR_LAST_WQE_REACH: + case UDMA_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: + case UDMA_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: + udma_qp_event(udma_dev, queue_num, event_type); + break; + case UDMA_EVENT_TYPE_MB: + udma_cmd_event(udma_dev, + le16_to_cpu(aeqe->event.cmd.token), + aeqe->event.cmd.status, + le64_to_cpu(aeqe->event.cmd.out_param)); + break; + default: + dev_err(udma_dev->dev, + "Unhandled event %d on EQ %d at idx %u.\n", + event_type, irq_work->eqn, irq_work->eq_ci); + break; + } +} + +static void udma_irq_work_handle(struct work_struct *work) +{ + struct udma_work *irq_work = + container_of(work, struct udma_work, work); + struct device *dev = irq_work->udma_dev->dev; + + aeq_event_dump(dev, irq_work); + aeq_event_report(irq_work->udma_dev, irq_work); + + kfree(irq_work); +} + +static void udma_init_irq_work(struct udma_dev *udma_dev, struct udma_eq *eq, + struct udma_aeqe *aeqe, uint32_t queue_num) +{ + struct udma_work *irq_work; + + irq_work = kzalloc(sizeof(struct udma_work), + GFP_ATOMIC); + if (!irq_work) + return; + + irq_work->udma_dev = udma_dev; + irq_work->event_type = eq->event_type; + irq_work->sub_type = eq->sub_type; + irq_work->queue_num = queue_num; + irq_work->eq_ci = eq->cons_index; + irq_work->eqn = eq->eqn; + + memcpy(&irq_work->aeqe, aeqe, sizeof(struct udma_aeqe)); + + INIT_WORK(&irq_work->work, udma_irq_work_handle); + queue_work(udma_dev->irq_workq, &irq_work->work); +} + +static inline void udma_write64(struct udma_dev *udma_dev, uint32_t val[2], + void __iomem *dest) +{ + struct udma_priv *priv = (struct udma_priv *)udma_dev->priv; + struct hnae3_handle *handle = priv->handle; + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + + if (!udma_dev->dis_db && !ops->get_hw_reset_stat(handle)) + writeq(*(uint64_t *)val, dest); +} + +static void update_eq_db(struct udma_eq *eq) +{ + struct udma_dev *udma_dev = eq->udma_dev; + struct udma_eq_db eq_db = {}; + + if (eq->type_flag == UDMA_AEQ) { + udma_reg_write(&eq_db, UDMA_EQ_DB_CMD, + eq->arm_st == UDMA_EQ_ALWAYS_ARMED ? + UDMA_EQ_DB_CMD_AEQ : + UDMA_EQ_DB_CMD_AEQ_ARMED); + } else { + udma_reg_write(&eq_db, UDMA_EQ_DB_TAG, eq->eqn); + + udma_reg_write(&eq_db, UDMA_EQ_DB_CMD, + eq->arm_st == UDMA_EQ_ALWAYS_ARMED ? + UDMA_EQ_DB_CMD_CEQ : + UDMA_EQ_DB_CMD_CEQ_ARMED); + } + + udma_reg_write(&eq_db, UDMA_EQ_DB_CI, eq->cons_index); + + udma_write64(udma_dev, (uint32_t *)&eq_db, eq->db_reg); +} + +static int udma_aeq_int(struct udma_dev *udma_dev, struct udma_eq *eq) +{ + struct udma_aeqe *aeqe = next_aeqe_sw_v2(eq); + int aeqe_found = 0; + uint32_t queue_num; + int event_type; + uint32_t *tmp; + int sub_type; + + while (aeqe) { + /* Make sure we read AEQ entry after we have checked the + * ownership bit + */ + dma_rmb(); + + event_type = udma_get_field(aeqe->asyn, + UDMA_AEQE_EVENT_TYPE_M, + UDMA_AEQE_EVENT_TYPE_S); + sub_type = udma_get_field(aeqe->asyn, + UDMA_AEQE_SUB_TYPE_M, + UDMA_AEQE_SUB_TYPE_S); + queue_num = udma_get_field(aeqe->event.queue_event.num, + UDMA_AEQE_EVENT_QUEUE_NUM_M, + UDMA_AEQE_EVENT_QUEUE_NUM_S); + tmp = (uint32_t *)aeqe; + if (event_type != UDMA_EVENT_TYPE_COMM_EST) + dev_err(udma_dev->dev, "print AEQE: 0x%x, 0x%x; " + "queue:0x%x event_type:0x%x, sub_type:0x%x\n", + *tmp, *(tmp + 1), queue_num, event_type, + sub_type); + + eq->event_type = event_type; + eq->sub_type = sub_type; + aeqe_found = 1; + + ++eq->cons_index; + + BUILD_BUG_ON(sizeof(struct udma_aeqe) > TRACE_AEQE_LEN_MAX); + + udma_init_irq_work(udma_dev, eq, aeqe, queue_num); + + aeqe = next_aeqe_sw_v2(eq); + } + + update_eq_db(eq); + return aeqe_found; +} + +static struct udma_ceqe *next_ceqe_sw_v2(struct udma_eq *eq) +{ + struct udma_ceqe *ceqe; + + ceqe = (struct udma_ceqe *)udma_buf_offset(eq->mtr.kmem, + (eq->cons_index & (eq->entries - 1)) * + eq->eqe_size); + + return (!!(udma_get_bit(ceqe->comp, UDMA_CEQ_CEQE_OWNER_S))) ^ + (!!(eq->cons_index & eq->entries)) ? ceqe : NULL; +} + +static int udma_ceq_int(struct udma_dev *udma_dev, + struct udma_eq *eq) +{ + struct udma_ceqe *ceqe = next_ceqe_sw_v2(eq); + int ceqe_found = 0; + uint32_t cqn; + + while (ceqe) { + /* Make sure we read CEQ entry after we have checked the + * ownership bit + */ + dma_rmb(); + + cqn = udma_get_field(ceqe->comp, UDMA_CEQE_COMP_CQN_M, + UDMA_CEQE_COMP_CQN_S); + + ++eq->cons_index; + ceqe_found = 1; + + ceqe = next_ceqe_sw_v2(eq); + } + + update_eq_db(eq); + + return ceqe_found; +} + +static irqreturn_t abnormal_interrupt_basic(struct udma_dev *udma_dev, + uint32_t int_st) +{ + struct pci_dev *pdev = udma_dev->pci_dev; + struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev); + const struct hnae3_ae_ops *ops = ae_dev->ops; + irqreturn_t int_work = IRQ_NONE; + uint32_t int_en; + + int_en = ub_read(udma_dev, UDMA_VF_ABN_INT_EN_REG); + if (int_st & BIT(UDMA_VF_INT_ST_AEQ_OVERFLOW_S)) { + dev_err(udma_dev->dev, "AEQ overflow!\n"); + + ub_write(udma_dev, UDMA_VF_ABN_INT_ST_REG, + 1 << UDMA_VF_INT_ST_AEQ_OVERFLOW_S); + + /* Set reset level for reset_event() */ + if (ops->set_default_reset_request) + ops->set_default_reset_request(ae_dev, + HNAE3_FUNC_RESET); + if (ops->reset_event) + ops->reset_event(pdev, NULL); + + int_en |= 1 << UDMA_VF_ABN_INT_EN_S; + ub_write(udma_dev, UDMA_VF_ABN_INT_EN_REG, int_en); + + int_work = IRQ_HANDLED; + } else { + dev_err(udma_dev->dev, "there is no basic abn irq found.\n"); + } + + return IRQ_RETVAL(int_work); +} + +static irqreturn_t udma_msix_interrupt_abn(int irq, void *dev_id) +{ + struct udma_dev *udma_dev = (struct udma_dev *)dev_id; + int int_work = 0; + uint32_t int_st; + + int_st = ub_read(udma_dev, UDMA_VF_ABN_INT_ST_REG); + if (int_st) { + int_work = abnormal_interrupt_basic(udma_dev, int_st); + } else { + dev_err(udma_dev->dev, "ECC ERROR not supported yet\n"); + int_work = IRQ_HANDLED; + } + + return IRQ_RETVAL(int_work); +} + +static irqreturn_t udma_msix_interrupt_eq(int irq, void *eq_ptr) +{ + struct udma_eq *eq = (struct udma_eq *)eq_ptr; + struct udma_dev *udma_dev = eq->udma_dev; + int int_work; + + if (eq->type_flag == UDMA_CEQ) + /* Completion event interrupt */ + int_work = udma_ceq_int(udma_dev, eq); + else + /* Asychronous event interrupt */ + int_work = udma_aeq_int(udma_dev, eq); + + return IRQ_RETVAL(int_work); +} + +static int alloc_and_set_irq_name(struct udma_dev *udma_dev, int irq_num, + int aeq_num, int other_num) +{ + int ret = 0; + int i; + + for (i = 0; i < irq_num; i++) { + udma_dev->irq_names[i] = kzalloc(UDMA_INT_NAME_LEN, GFP_KERNEL); + if (!udma_dev->irq_names[i]) { + ret = -ENOMEM; + goto err_kzalloc_failed; + } + } + + /* irq contains: abnormal + AEQ + CEQ */ + for (i = 0; i < other_num; i++) + snprintf((char *)udma_dev->irq_names[i], UDMA_INT_NAME_LEN, + "udma-abn-%d", i); + + for (i = other_num; i < (other_num + aeq_num); i++) + snprintf((char *)udma_dev->irq_names[i], UDMA_INT_NAME_LEN, + "udma-aeq-%d", i - other_num); + + for (i = (other_num + aeq_num); i < irq_num; i++) + snprintf((char *)udma_dev->irq_names[i], UDMA_INT_NAME_LEN, + "udma-ceq-%d", i - other_num - aeq_num); + return ret; + +err_kzalloc_failed: + for (i -= 1; i >= 0; i--) + kfree(udma_dev->irq_names[i]); + + return ret; +} + +static int udma_request_irq(struct udma_dev *udma_dev, int irq_num, + int ceq_num, int aeq_num, int other_num) +{ + struct udma_eq_table *eq_table = &udma_dev->eq_table; + int ret; + int j; + + ret = alloc_and_set_irq_name(udma_dev, irq_num, aeq_num, other_num); + if (ret) + return ret; + for (j = 0; j < irq_num; j++) { + if (j < other_num) + ret = request_irq(udma_dev->irq[j], + udma_msix_interrupt_abn, + 0, udma_dev->irq_names[j], udma_dev); + + else if (j < (other_num + ceq_num)) + ret = request_irq(eq_table->eq[j - other_num].irq, + udma_msix_interrupt_eq, + 0, udma_dev->irq_names[j + aeq_num], + &eq_table->eq[j - other_num]); + else + ret = request_irq(eq_table->eq[j - other_num].irq, + udma_msix_interrupt_eq, + 0, udma_dev->irq_names[j - ceq_num], + &eq_table->eq[j - other_num]); + if (ret) { + dev_err(udma_dev->dev, "Request irq error!\n"); + goto err_request_failed; + } + } + + return 0; + +err_request_failed: + for (j -= 1; j >= 0; j--) + if (j < other_num) + free_irq(udma_dev->irq[j], udma_dev); + else + free_irq(eq_table->eq[j - other_num].irq, + &eq_table->eq[j - other_num]); + for (j = irq_num - 1; j >= 0; j--) + kfree(udma_dev->irq_names[j]); + + return ret; +} + +static void udma_int_mask_enable(struct udma_dev *udma_dev, + int eq_num, uint32_t enable_flag) +{ + int i; + + for (i = 0; i < eq_num; i++) + ub_write(udma_dev, UDMA_VF_EVENT_INT_EN_REG + + i * EQ_REG_OFFSET, enable_flag); + + ub_write(udma_dev, UDMA_VF_ABN_INT_EN_REG, enable_flag); + ub_write(udma_dev, UDMA_VF_ABN_INT_CFG_REG, enable_flag); +} + +static void set_jfce_attr(struct udma_dev *udma_dev, struct udma_eq *eq, + int idx) +{ + eq->type_flag = UDMA_CEQ; + eq->entries = udma_dev->caps.ceqe_depth; + eq->eqe_size = udma_dev->caps.ceqe_size; + eq->irq = udma_dev->irq[idx]; + eq->eq_max_cnt = UDMA_CEQ_DEFAULT_BURST_NUM; + eq->eq_period = UDMA_CEQ_DEFAULT_INTERVAL; +} + +static void set_jfae_attr(struct udma_dev *udma_dev, struct udma_eq *eq, + int idx) +{ + eq->type_flag = UDMA_AEQ; + eq->entries = udma_dev->caps.aeqe_depth; + eq->eqe_size = udma_dev->caps.aeqe_size; + eq->irq = udma_dev->irq[idx]; + eq->eq_max_cnt = UDMA_AEQ_DEFAULT_BURST_NUM; + eq->eq_period = UDMA_AEQ_DEFAULT_INTERVAL; +} + +static int udma_create_hw_eq(struct udma_dev *udma_dev, int ceq_num, + int aeq_num, int other_num) +{ + struct udma_eq_table *eq_table = &udma_dev->eq_table; + struct udma_eq *eq; + uint32_t eq_cmd; + int eq_num; + int i; + int ret; + + eq_num = ceq_num + aeq_num; + + for (i = 0; i < eq_num; i++) { + eq = &eq_table->eq[i]; + eq->udma_dev = udma_dev; + eq->eqn = i; + if (i < ceq_num) { + /* JFCE */ + eq_cmd = UDMA_CMD_CREATE_CEQC; + set_jfce_attr(udma_dev, eq, (i + other_num + aeq_num)); + } else { + /* JFAE */ + eq_cmd = UDMA_CMD_CREATE_AEQC; + set_jfae_attr(udma_dev, eq, (i - ceq_num + other_num)); + } + init_eq_config(udma_dev, eq); + ret = alloc_eq_buf(udma_dev, eq); + if (ret) { + dev_err(udma_dev->dev, "failed to alloc eq buf.\n"); + goto err_out; + } + + ret = udma_create_eq(udma_dev, eq, eq_cmd); + if (ret) { + dev_err(udma_dev->dev, "failed to create eq.\n"); + free_eq_buf(udma_dev, &eq_table->eq[i]); + goto err_out; + } + } + return ret; + +err_out: + for (i -= 1; i >= 0; i--) + free_eq_buf(udma_dev, &eq_table->eq[i]); + + return ret; +} + +void udma_destroy_eqc(struct udma_dev *udma_dev, int eqn, uint32_t eq_cmd) +{ + struct device *dev = udma_dev->dev; + struct udma_cmq_desc desc; + struct udma_mbox *mb; + int ret; + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + + mbox_desc_init(mb, 0, 0, eqn & UDMA_EQN_M, eq_cmd); + + ret = udma_cmd_mbox(udma_dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); + if (ret) + dev_err(dev, "[mailbox cmd] destroy eqc(%d) failed.\n", eqn); +} + +static void __udma_free_irq(struct udma_dev *udma_dev) +{ + int irq_num; + int eq_num; + int i; + + eq_num = udma_dev->caps.num_comp_vectors + + udma_dev->caps.num_aeq_vectors; + irq_num = eq_num + udma_dev->caps.num_other_vectors; + + for (i = 0; i < udma_dev->caps.num_other_vectors; i++) + free_irq(udma_dev->irq[i], udma_dev); + + for (i = 0; i < eq_num; i++) + free_irq(udma_dev->eq_table.eq[i].irq, + &udma_dev->eq_table.eq[i]); + + for (i = 0; i < irq_num; i++) + kfree(udma_dev->irq_names[i]); +} + +void udma_cleanup_eq_table(struct udma_dev *udma_dev) +{ + struct udma_eq_table *eq_table = &udma_dev->eq_table; + uint32_t eq_cmd; + int eq_num; + int i; + + eq_num = udma_dev->caps.num_comp_vectors + + udma_dev->caps.num_aeq_vectors; + + /* Disable irq */ + udma_int_mask_enable(udma_dev, eq_num, EQ_DISABLE); + + __udma_free_irq(udma_dev); + flush_workqueue(udma_dev->irq_workq); + destroy_workqueue(udma_dev->irq_workq); + + for (i = 0; i < eq_num; i++) { + if (i < udma_dev->caps.num_comp_vectors) + eq_cmd = UDMA_CMD_DESTROY_CEQC; + else + eq_cmd = UDMA_CMD_DESTROY_AEQC; + + udma_destroy_eqc(udma_dev, i, eq_cmd); + free_eq_buf(udma_dev, &eq_table->eq[i]); + } + + kfree(eq_table->eq); +} + +int udma_init_eq_table(struct udma_dev *udma_dev) +{ + struct udma_eq_table *eq_table = &udma_dev->eq_table; + struct device *dev = udma_dev->dev; + int other_num; + int irq_num; + int ceq_num; + int aeq_num; + int eq_num; + int ret; + int i; + + other_num = udma_dev->caps.num_other_vectors; + ceq_num = udma_dev->caps.num_comp_vectors; + aeq_num = udma_dev->caps.num_aeq_vectors; + + eq_num = ceq_num + aeq_num; + irq_num = eq_num + other_num; + + eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL); + if (!eq_table->eq) + return -ENOMEM; + ret = udma_create_hw_eq(udma_dev, ceq_num, aeq_num, other_num); + if (ret) + goto err_create_eq_fail; + + udma_dev->irq_workq = alloc_ordered_workqueue("udma_irq_workq", 0); + if (!udma_dev->irq_workq) { + dev_err(dev, "failed to create irq workqueue.\n"); + ret = -ENOMEM; + goto err_alloc_workqueue_fail; + } + + ret = udma_request_irq(udma_dev, irq_num, ceq_num, aeq_num, + other_num); + if (ret) { + dev_err(dev, "failed to request irq.\n"); + goto err_request_irq_fail; + } + + /* enable irq */ + udma_int_mask_enable(udma_dev, eq_num, EQ_ENABLE); + + return 0; + +err_request_irq_fail: + destroy_workqueue(udma_dev->irq_workq); + +err_alloc_workqueue_fail: + for (i = ceq_num + aeq_num - 1; i >= 0; i--) + free_eq_buf(udma_dev, &eq_table->eq[i]); + +err_create_eq_fail: + kfree(eq_table->eq); + return ret; +} diff --git a/drivers/ub/hw/hns3/hns3_udma_eq.h b/drivers/ub/hw/hns3/hns3_udma_eq.h new file mode 100644 index 000000000000..b32a6b2e9f27 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_eq.h @@ -0,0 +1,104 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_EQ_H +#define _UDMA_EQ_H + +#include +#include "hns3_udma_cmd.h" + +struct udma_eq_context { + uint32_t data[16]; +}; + +#define UDMA_EQ_STATE_VALID 1 +#define UDMA_EQ_INIT_EQE_CNT 0 +#define UDMA_EQ_INIT_PROD_IDX 0 +#define UDMA_EQ_INIT_REPORT_TIMER 0 +#define UDMA_EQ_INIT_MSI_IDX 0 +#define UDMA_EQ_INIT_CONS_IDX 0 +#define UDMA_EQ_INIT_NXT_EQE_BA 0 + +#define EQC_FIELD_LOC(h, l) ((uint64_t)(h) << 32 | (l)) + +#define EQC_EQ_ST EQC_FIELD_LOC(1, 0) +#define EQC_EQE_HOP_NUM EQC_FIELD_LOC(3, 2) +#define EQC_OVER_IGNORE EQC_FIELD_LOC(4, 4) +#define EQC_COALESCE EQC_FIELD_LOC(5, 5) +#define EQC_ARM_ST EQC_FIELD_LOC(7, 6) +#define EQC_EQN EQC_FIELD_LOC(15, 8) +#define EQC_EQE_CNT EQC_FIELD_LOC(31, 16) +#define EQC_EQE_BA_PG_SZ EQC_FIELD_LOC(35, 32) +#define EQC_EQE_BUF_PG_SZ EQC_FIELD_LOC(39, 36) +#define EQC_EQ_PROD_INDX EQC_FIELD_LOC(63, 40) +#define EQC_EQ_MAX_CNT EQC_FIELD_LOC(79, 64) +#define EQC_EQ_PERIOD EQC_FIELD_LOC(95, 80) +#define EQC_EQE_REPORT_TIMER EQC_FIELD_LOC(127, 96) +#define EQC_EQE_BA_L EQC_FIELD_LOC(159, 128) +#define EQC_EQE_BA_H EQC_FIELD_LOC(188, 160) +#define EQC_SHIFT EQC_FIELD_LOC(199, 192) +#define EQC_MSI_INDX EQC_FIELD_LOC(207, 200) +#define EQC_CUR_EQE_BA_L EQC_FIELD_LOC(223, 208) +#define EQC_CUR_EQE_BA_M EQC_FIELD_LOC(255, 224) +#define EQC_CUR_EQE_BA_H EQC_FIELD_LOC(259, 256) +#define EQC_EQ_CONS_INDX EQC_FIELD_LOC(287, 264) +#define EQC_NEX_EQE_BA_L EQC_FIELD_LOC(319, 288) +#define EQC_NEX_EQE_BA_H EQC_FIELD_LOC(339, 320) +#define EQC_EQE_SIZE EQC_FIELD_LOC(341, 340) + +#define UDMA_CEQE_COMP_CQN_S 0 +#define UDMA_CEQE_COMP_CQN_M GENMASK(23, 0) + +#define UDMA_AEQE_EVENT_TYPE_S 0 +#define UDMA_AEQE_EVENT_TYPE_M GENMASK(7, 0) + +#define UDMA_AEQE_SUB_TYPE_S 8 +#define UDMA_AEQE_SUB_TYPE_M GENMASK(15, 8) + +#define UDMA_AEQE_EVENT_QUEUE_NUM_S 0 +#define UDMA_AEQE_EVENT_QUEUE_NUM_M GENMASK(23, 0) + +#define EQC_EQE_BA_L_SHIFT 3 +#define EQC_EQE_BA_H_SHIFT 35 +#define EQC_CUR_EQE_BA_L_SHIFT 12 +#define EQC_CUR_EQE_BA_M_SHIFT 28 +#define EQC_CUR_EQE_BA_H_SHIFT 60 +#define EQC_NEX_EQE_BA_L_SHIFT 12 +#define EQC_NEX_EQE_BA_H_SHIFT 44 +#define UDMA_VF_INT_ST_AEQ_OVERFLOW_S 0 +#define UDMA_VF_ABN_INT_EN_S 0 + +struct udma_eq_db { + uint32_t data[2]; +}; + +#define EQ_DB_FIELD_LOC(h, l) ((uint64_t)(h) << 32 | (l)) + +#define UDMA_EQ_DB_TAG EQ_DB_FIELD_LOC(7, 0) +#define UDMA_EQ_DB_CMD EQ_DB_FIELD_LOC(17, 16) +#define UDMA_EQ_DB_CI EQ_DB_FIELD_LOC(55, 32) + +#define UDMA_EQ_ARMED 1 +#define UDMA_EQ_ALWAYS_ARMED 3 + +#define UDMA_EQ_DB_CMD_AEQ 0x0 +#define UDMA_EQ_DB_CMD_AEQ_ARMED 0x1 +#define UDMA_EQ_DB_CMD_CEQ 0x2 +#define UDMA_EQ_DB_CMD_CEQ_ARMED 0x3 + +int udma_init_eq_table(struct udma_dev *udma_dev); +void udma_cleanup_eq_table(struct udma_dev *udma_dev); + +#endif /* _UDMA_EQ_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_hem.h b/drivers/ub/hw/hns3/hns3_udma_hem.h index 2eacd3fc7799..ddf1b9c74075 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hem.h +++ b/drivers/ub/hw/hns3/hns3_udma_hem.h @@ -18,6 +18,7 @@ #include #include +#include #include "hns3_udma_device.h" #define HEM_HOP_STEP_DIRECT 0xff @@ -126,6 +127,9 @@ int udma_table_get(struct udma_dev *udma_dev, struct udma_hem_table *table, uint64_t obj); void udma_table_put(struct udma_dev *udma_dev, struct udma_hem_table *table, uint64_t obj); +void *udma_table_find(struct udma_dev *udma_dev, + struct udma_hem_table *table, uint64_t obj, + dma_addr_t *dma_handle); int udma_init_hem_table(struct udma_dev *udma_dev, struct udma_hem_table *table, uint32_t type, uint64_t obj_size, uint64_t nobj); @@ -136,6 +140,15 @@ int udma_calc_hem_mhop(struct udma_dev *udma_dev, struct udma_hem_mhop *mhop); bool udma_check_whether_mhop(struct udma_dev *udma_dev, uint32_t type); +int udma_mtr_find(struct udma_dev *udma_device, struct udma_mtr *mtr, + int offset, uint64_t *mtt_buf, int mtt_max, + uint64_t *base_addr); + +void udma_mtr_move(struct udma_mtr *from_mtr, struct udma_mtr *to_mtr); + +int udma_mtr_map(struct udma_dev *dev, struct udma_mtr *mtr, + dma_addr_t *pages, uint32_t page_count); + static inline void udma_hem_first(struct udma_hem *hem, struct udma_hem_iter *iter) { diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.c b/drivers/ub/hw/hns3/hns3_udma_hw.c index 0e971a2649ab..936777ec26e6 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hw.c +++ b/drivers/ub/hw/hns3/hns3_udma_hw.c @@ -19,6 +19,8 @@ #include "hnae3.h" #include "hns3_udma_cmd.h" #include "hns3_udma_hem.h" +#include "hns3_udma_eq.h" +#include "hns3_udma_qp.h" static const struct pci_device_id udma_hw_pci_tbl[] = { { PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_UDMA_OVER_UBL), @@ -1454,6 +1456,8 @@ static const struct udma_hw udma_hw = { .poll_mbox_done = udma_poll_mbox_done, .set_hem = udma_set_hem, .clear_hem = udma_clear_hem, + .init_eq = udma_init_eq_table, + .cleanup_eq = udma_cleanup_eq_table, }; bool udma_is_virtual_func(struct pci_dev *pdev) diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.h b/drivers/ub/hw/hns3/hns3_udma_hw.h index 19b69496d82e..250e78631c86 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hw.h +++ b/drivers/ub/hw/hns3/hns3_udma_hw.h @@ -81,6 +81,14 @@ struct udma_query_oor_cmq { #define UDMA_CQC_TIMER_ENTRY_SZ PAGE_SIZE #define UDMA_EQE_SIZE 0x40 +#define UDMA_EQ_OVER_IGNORE_0 0 +#define UDMA_EQ_COALESCE_0 0 +#define UDMA_EQ_ALWAYS_ARMED 3 +#define UDMA_EQN_M GENMASK(23, 0) + +#define UDMA_CEQ_CEQE_OWNER_S 31 +#define UDMA_AEQ_AEQE_OWNER_S 31 + #define NIC_ICL_SWITCH_CMD_UDMA_SEL_SHIFT 0 #define NIC_ICL_SWITCH_CMD_UDMA_SEL BIT(NIC_ICL_SWITCH_CMD_UDMA_SEL_SHIFT) @@ -390,4 +398,6 @@ enum { #define CFG_GMV_BT_IDX CMQ_REQ_FIELD_LOC(95, 64) #define CFG_GMV_BT_VF_ID CMQ_REQ_FIELD_LOC(103, 96) +#define UDMA_INT_NAME_LEN 32 + #endif /* _UDMA_HW_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c index 134a2b8def69..47d66dc8fdb5 100644 --- a/drivers/ub/hw/hns3/hns3_udma_main.c +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -511,6 +511,12 @@ int udma_hnae_client_init(struct udma_dev *udma_dev) goto error_failed_cmd_init; } + ret = udma_dev->hw->init_eq(udma_dev); + if (ret) { + dev_err(dev, "eq init failed!\n"); + goto error_failed_eq_table; + } + if (udma_dev->cmd_mod) { ret = udma_cmd_use_events(udma_dev); if (ret) { @@ -558,6 +564,9 @@ int udma_hnae_client_init(struct udma_dev *udma_dev) if (udma_dev->cmd_mod) udma_cmd_use_polling(udma_dev); + udma_dev->hw->cleanup_eq(udma_dev); + +error_failed_eq_table: udma_cmd_cleanup(udma_dev); error_failed_cmd_init: @@ -582,6 +591,8 @@ void udma_hnae_client_exit(struct udma_dev *udma_dev) if (udma_dev->cmd_mod) udma_cmd_use_polling(udma_dev); + udma_dev->hw->cleanup_eq(udma_dev); + udma_cmd_cleanup(udma_dev); if (udma_dev->hw->cmq_exit) udma_dev->hw->cmq_exit(udma_dev); diff --git a/drivers/ub/hw/hns3/hns3_udma_qp.c b/drivers/ub/hw/hns3/hns3_udma_qp.c index 609d93b2a085..2258149dfb1b 100644 --- a/drivers/ub/hw/hns3/hns3_udma_qp.c +++ b/drivers/ub/hw/hns3/hns3_udma_qp.c @@ -16,6 +16,7 @@ #include #include "hns3_udma_jfc.h" #include "hns3_udma_hem.h" +#include "hns3_udma_qp.h" static inline uint8_t get_qp_bankid(uint64_t qpn) { /* The lower 3 bits of QPN are used to hash to different banks */ @@ -61,3 +62,26 @@ void udma_cleanup_qp_table(struct udma_dev *dev) ida_destroy(&dev->qp_table.bank[i].ida); kfree(dev->qp_table.idx_table.spare_idx); } + +void udma_qp_event(struct udma_dev *udma_dev, uint32_t qpn, int event_type) +{ + struct device *dev = udma_dev->dev; + struct udma_qp *qp; + + xa_lock(&udma_dev->qp_table.xa); + qp = (struct udma_qp *)xa_load(&udma_dev->qp_table.xa, qpn); + if (qp) + refcount_inc(&qp->refcount); + xa_unlock(&udma_dev->qp_table.xa); + + if (!qp) { + dev_warn(dev, "Async event for bogus QP 0x%08x\n", qpn); + return; + } + + if (qp->event) + qp->event(qp, (enum udma_event)event_type); + + if (refcount_dec_and_test(&qp->refcount)) + complete(&qp->free); +} diff --git a/drivers/ub/hw/hns3/hns3_udma_qp.h b/drivers/ub/hw/hns3/hns3_udma_qp.h new file mode 100644 index 000000000000..bd0be4cb27f1 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_qp.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_QP_H +#define _UDMA_QP_H + +#include "hns3_udma_device.h" + +struct udma_qp { + struct udma_dev *udma_device; + struct udma_mtr mtr; + void (*event)(struct udma_qp *qp, + enum udma_event event_type); + uint64_t qpn; + + refcount_t refcount; + struct completion free; +}; + +void udma_qp_event(struct udma_dev *udma_dev, uint32_t qpn, int event_type); + +#endif /* _UDMA_QP_H */ -- Gitee From e63f303bdc26bfddefaba5c322fba46624a5b7fa Mon Sep 17 00:00:00 2001 From: Weibo Zhao Date: Tue, 15 Aug 2023 17:09:21 +0800 Subject: [PATCH 07/42] hns3 udma: create and destroy u-context driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA ------------------------------------------------------------ This patch is code for create and destroy u-context. Context is isolated by process. Uar in u-context contains doorbell address and direct wqe address. Signed-off-by: Weibo Zhao --- drivers/ub/hw/hns3/hns3_udma_abi.h | 34 ++++++++ drivers/ub/hw/hns3/hns3_udma_device.h | 28 +++++++ drivers/ub/hw/hns3/hns3_udma_main.c | 112 +++++++++++++++++++++++++- 3 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 drivers/ub/hw/hns3/hns3_udma_abi.h diff --git a/drivers/ub/hw/hns3/hns3_udma_abi.h b/drivers/ub/hw/hns3/hns3_udma_abi.h new file mode 100644 index 000000000000..e0c260da6c5b --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_abi.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_ABI_H +#define _UDMA_ABI_H + +#include +struct udma_create_ctx_resp { + uint32_t num_comp_vectors; + uint32_t num_qps_shift; + uint32_t num_jfs_shift; + uint32_t num_jfr_shift; + uint32_t num_jetty_shift; + uint32_t max_jfc_cqe; + uint32_t cqe_size; + uint32_t max_jfr_wr; + uint32_t max_jfr_sge; + uint32_t max_jfs_wr; + uint32_t max_jfs_sge; +}; + +#endif /* _UDMA_ABI_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_device.h b/drivers/ub/hw/hns3/hns3_udma_device.h index 76074f264955..6bea23607708 100644 --- a/drivers/ub/hw/hns3/hns3_udma_device.h +++ b/drivers/ub/hw/hns3/hns3_udma_device.h @@ -78,6 +78,10 @@ #define EQ_REG_OFFSET 0x4 #define UDMA_DEFAULT_MAX_JETTY_X_SHIFT 8 +#define UDMA_DB_ADDR_OFFSET 0x230 +#define UDMA_DEV_START_OFFSET 2 +#define UDMA_DEV_EX_START_OFFSET 4 + enum { NO_ARMED = 0x0 }; @@ -138,6 +142,12 @@ enum { UDMA_BUF_NOFAIL = BIT(2), }; +struct udma_uar { + uint64_t pfn; + uint64_t index; + uint64_t logic_idx; +}; + struct udma_ida { struct ida ida; uint32_t min; /* Lowest ID to allocate. */ @@ -241,6 +251,12 @@ struct udma_work { }; struct udma_dev; +struct udma_ucontext { + struct ubcore_ucontext uctx; + struct udma_uar uar; + uint64_t pdn; +}; + struct udma_cmd_context { struct completion done; int result; @@ -621,6 +637,7 @@ struct udma_dev { struct udma_hem_table qpc_timer_table; struct udma_hem_table cqc_timer_table; struct udma_hem_table gmv_table; + uint64_t dwqe_page; struct list_head qp_list; spinlock_t qp_list_lock; struct list_head dip_list; @@ -652,6 +669,17 @@ static inline uint32_t to_udma_hem_hopnum(uint32_t hopnum, uint32_t count) return 0; } +static inline struct udma_ucontext + *to_udma_ucontext(struct ubcore_ucontext *uctx) +{ + return container_of(uctx, struct udma_ucontext, uctx); +} + +static inline struct udma_dev *to_udma_dev(const struct ubcore_device *ubcore_dev) +{ + return container_of(ubcore_dev, struct udma_dev, ub_dev); +} + static inline uint32_t to_udma_hw_page_shift(uint32_t page_shift) { return page_shift - UDMA_HW_PAGE_SHIFT; diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c index 47d66dc8fdb5..ed1841a22d47 100644 --- a/drivers/ub/hw/hns3/hns3_udma_main.c +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -13,14 +13,124 @@ * */ +#include #include "urma/ubcore_api.h" +#include "hns3_udma_abi.h" #include "hns3_udma_device.h" #include "hns3_udma_hem.h" #include "hns3_udma_jfc.h" #include "hns3_udma_cmd.h" +static int udma_uar_alloc(struct udma_dev *udma_dev, struct udma_uar *uar) +{ + struct udma_ida *uar_ida = &udma_dev->uar_ida; + int id; + + /* Using bitmap to manager UAR index */ + id = ida_alloc_range(&uar_ida->ida, uar_ida->min, uar_ida->max, + GFP_KERNEL); + if (id < 0) { + dev_err(udma_dev->dev, "failed to alloc uar id(%d).\n", id); + return id; + } + uar->logic_idx = (uint64_t)id; -static struct ubcore_ops g_udma_dev_ops = { + if (uar->logic_idx > 0 && udma_dev->caps.phy_num_uars > 1) + uar->index = (uar->logic_idx - 1) % + (udma_dev->caps.phy_num_uars - 1) + 1; + else + uar->index = 0; + + uar->pfn = ((pci_resource_start(udma_dev->pci_dev, + UDMA_DEV_START_OFFSET)) >> PAGE_SHIFT); + if (udma_dev->caps.flags & UDMA_CAP_FLAG_DIRECT_WQE) + udma_dev->dwqe_page = + pci_resource_start(udma_dev->pci_dev, + UDMA_DEV_EX_START_OFFSET); + return 0; +} + +static int udma_init_ctx_resp(struct udma_dev *dev, struct ubcore_udrv_priv *udrv_data) +{ + struct udma_create_ctx_resp resp = {}; + int ret; + + resp.num_comp_vectors = dev->caps.num_comp_vectors; + resp.num_qps_shift = dev->caps.num_qps_shift; + resp.num_jfs_shift = dev->caps.num_jfs_shift; + resp.num_jfr_shift = dev->caps.num_jfr_shift; + resp.num_jetty_shift = dev->caps.num_jetty_shift; + resp.max_jfc_cqe = dev->caps.max_cqes; + resp.cqe_size = dev->caps.cqe_sz; + resp.max_jfr_wr = dev->caps.max_srq_wrs; + resp.max_jfr_sge = dev->caps.max_srq_sges; + resp.max_jfs_wr = dev->caps.max_wqes; + resp.max_jfs_sge = dev->caps.max_sq_sg; + + ret = copy_to_user((void *)udrv_data->out_addr, &resp, + min(udrv_data->out_len, (uint32_t)sizeof(resp))); + if (ret) + dev_err(dev->dev, + "copy ctx resp to user failed, ret = %d.\n", ret); + + return ret; +} + +static void udma_uar_free(struct udma_dev *udma_dev, + struct udma_ucontext *context) +{ + ida_free(&udma_dev->uar_ida.ida, (int)context->uar.logic_idx); +} + +static struct ubcore_ucontext *udma_alloc_ucontext(struct ubcore_device *dev, + uint32_t uasid, + struct ubcore_udrv_priv *udrv_data) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + struct udma_ucontext *context; + int ret; + + context = kzalloc(sizeof(struct udma_ucontext), GFP_KERNEL); + if (!context) + return NULL; + + ret = udma_uar_alloc(udma_dev, &context->uar); + if (ret) { + dev_err(udma_dev->dev, "Alloc udma_uar Failed.\n"); + goto err_alloc_ucontext; + } + + context->pdn = uasid; /* Use the UASID as pd number */ + ret = udma_init_ctx_resp(udma_dev, udrv_data); + if (ret) { + dev_err(udma_dev->dev, "Init ctx resp failed.\n"); + goto err_alloc_uar; + } + + return &context->uctx; + +err_alloc_uar: + udma_uar_free(udma_dev, context); +err_alloc_ucontext: + kfree(context); + return NULL; +} + +static int udma_free_ucontext(struct ubcore_ucontext *uctx) +{ + struct udma_ucontext *context = to_udma_ucontext(uctx); + struct udma_dev *udma_dev = to_udma_dev(uctx->ub_dev); + + ida_free(&udma_dev->uar_ida.ida, (int)context->uar.logic_idx); + kfree(context); + return 0; +} + +static struct ubcore_ops g_udma_dev_ops = { + .owner = THIS_MODULE, + .abi_version = 1, + .alloc_ucontext = udma_alloc_ucontext, + .free_ucontext = udma_free_ucontext, }; static void udma_cleanup_uar_table(struct udma_dev *dev) -- Gitee From 6060aa7bd66a19185f6c9cd586996ba26468b94e Mon Sep 17 00:00:00 2001 From: Weibo Zhao Date: Tue, 15 Aug 2023 20:20:28 +0800 Subject: [PATCH 08/42] hns3 udma: mmap doorbell address to uar page driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA ----------------------------------------------- This patch mmap doorbell address of hardware to uar page. Doorbell address is in bar23 area. Signed-off-by: Weibo Zhao --- drivers/ub/hw/hns3/hns3_udma_abi.h | 7 ++++++ drivers/ub/hw/hns3/hns3_udma_main.c | 35 +++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/drivers/ub/hw/hns3/hns3_udma_abi.h b/drivers/ub/hw/hns3/hns3_udma_abi.h index e0c260da6c5b..358a57c78bfd 100644 --- a/drivers/ub/hw/hns3/hns3_udma_abi.h +++ b/drivers/ub/hw/hns3/hns3_udma_abi.h @@ -17,6 +17,13 @@ #define _UDMA_ABI_H #include + +#define MAP_COMMAND_MASK 0xff + +enum { + UDMA_MMAP_UAR_PAGE, +}; + struct udma_create_ctx_resp { uint32_t num_comp_vectors; uint32_t num_qps_shift; diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c index ed1841a22d47..34e87b1e8864 100644 --- a/drivers/ub/hw/hns3/hns3_udma_main.c +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -13,6 +13,7 @@ * */ +#include #include #include "urma/ubcore_api.h" #include "hns3_udma_abi.h" @@ -126,11 +127,45 @@ static int udma_free_ucontext(struct ubcore_ucontext *uctx) return 0; } +static int get_mmap_cmd(struct vm_area_struct *vma) +{ + return (vma->vm_pgoff & MAP_COMMAND_MASK); +} + +static int udma_mmap(struct ubcore_ucontext *uctx, struct vm_area_struct *vma) +{ + struct udma_dev *udma_dev = to_udma_dev(uctx->ub_dev); + int cmd; + + if (((vma->vm_end - vma->vm_start) % PAGE_SIZE) != 0) { + dev_err(udma_dev->dev, + "mmap failed, unexpected vm area size.\n"); + return -EINVAL; + } + + cmd = get_mmap_cmd(vma); + switch (cmd) { + case UDMA_MMAP_UAR_PAGE: + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + if (io_remap_pfn_range(vma, vma->vm_start, + to_udma_ucontext(uctx)->uar.pfn, + PAGE_SIZE, vma->vm_page_prot)) + return -EAGAIN; + break; + default: + dev_err(udma_dev->dev, + "mmap failed, cmd(%d) not support\n", cmd); + return -EINVAL; + } + + return 0; +} static struct ubcore_ops g_udma_dev_ops = { .owner = THIS_MODULE, .abi_version = 1, .alloc_ucontext = udma_alloc_ucontext, .free_ucontext = udma_free_ucontext, + .mmap = udma_mmap, }; static void udma_cleanup_uar_table(struct udma_dev *dev) -- Gitee From 5918be3e281ef20f573a7dcc91c488b609d55ed6 Mon Sep 17 00:00:00 2001 From: Weibo Zhao Date: Wed, 16 Aug 2023 09:55:42 +0800 Subject: [PATCH 09/42] hns3 udma: feature of record doorbell driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA -------------------------------------------------------------- This patch is the code for record doorbell. Record doorbell is a 8B memory which saves value of jfr pi and jfc ci. It works as doorbell but not real-time. When hardware need to update jfr-pi for example, hardware would read jfr record doorbell to update latest jfr pi value. Signed-off-by: Weibo Zhao --- drivers/ub/hw/hns3/Makefile | 3 +- drivers/ub/hw/hns3/hns3_udma_db.c | 83 +++++++++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_db.h | 26 +++++++++ drivers/ub/hw/hns3/hns3_udma_device.h | 15 +++++ 4 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 drivers/ub/hw/hns3/hns3_udma_db.c create mode 100644 drivers/ub/hw/hns3/hns3_udma_db.h diff --git a/drivers/ub/hw/hns3/Makefile b/drivers/ub/hw/hns3/Makefile index 2cae99de3143..38f7b84cf7e1 100644 --- a/drivers/ub/hw/hns3/Makefile +++ b/drivers/ub/hw/hns3/Makefile @@ -8,6 +8,7 @@ ccflags-y += -I$(srctree)/drivers/net/ethernet/hisilicon/hns3/ \ -I$(srctree)/drivers/ub/urma/ $(MODULE_NAME)-objs := hns3_udma_hw.o hns3_udma_main.o hns3_udma_cmd.o \ - hns3_udma_hem.o hns3_udma_qp.o hns3_udma_eq.o + hns3_udma_hem.o hns3_udma_qp.o hns3_udma_eq.o \ + hns3_udma_db.o obj-$(CONFIG_UB_UDMA_HNS3) := hns3_udma.o diff --git a/drivers/ub/hw/hns3/hns3_udma_db.c b/drivers/ub/hw/hns3/hns3_udma_db.c new file mode 100644 index 000000000000..0880d4d4bcfd --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_db.c @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include +#include +#include "urma/ubcore_api.h" +#include "hns3_udma_device.h" +#include "hns3_udma_db.h" + +int udma_db_map_user(struct udma_dev *udma_dev, uint64_t virt, + struct udma_db *db) +{ + uint64_t page_addr = virt & PAGE_MASK; + union ubcore_umem_flag access = {}; + struct udma_user_db_page *db_page; + uint32_t offset; + int ret = 0; + + mutex_lock(&udma_dev->pgdir_mutex); + + list_for_each_entry(db_page, &udma_dev->pgdir_list, list) { + if (db_page->user_virt == page_addr) + goto found; + } + + db_page = kmalloc(sizeof(*db_page), GFP_KERNEL); + if (!db_page) { + ret = -ENOMEM; + goto out; + } + + refcount_set(&db_page->refcount, 1); + db_page->user_virt = page_addr; + access.bs.non_pin = 0; + access.bs.writable = 1; + db_page->umem = ubcore_umem_get(&udma_dev->ub_dev, page_addr, + PAGE_SIZE, access); + if (IS_ERR(db_page->umem)) { + ret = PTR_ERR(db_page->umem); + kfree(db_page); + goto out; + } + + list_add(&db_page->list, &udma_dev->pgdir_list); + +found: + offset = virt - page_addr; + db->dma = sg_dma_address(db_page->umem->sg_head.sgl) + offset; + db->virt_addr = (char *)sg_virt(db_page->umem->sg_head.sgl) + offset; + db->user_page = db_page; + refcount_inc(&db_page->refcount); + +out: + mutex_unlock(&udma_dev->pgdir_mutex); + + return ret; +} + +void udma_db_unmap_user(struct udma_dev *udma_dev, struct udma_db *db) +{ + mutex_lock(&udma_dev->pgdir_mutex); + + refcount_dec(&db->user_page->refcount); + if (refcount_dec_if_one(&db->user_page->refcount)) { + list_del(&db->user_page->list); + ubcore_umem_release(db->user_page->umem); + kfree(db->user_page); + } + + mutex_unlock(&udma_dev->pgdir_mutex); +} diff --git a/drivers/ub/hw/hns3/hns3_udma_db.h b/drivers/ub/hw/hns3/hns3_udma_db.h new file mode 100644 index 000000000000..e5a230cf49c0 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_db.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_DB_H +#define _UDMA_DB_H + +#include "hns3_udma_device.h" + +int udma_db_map_user(struct udma_dev *udma_dev, uint64_t virt, + struct udma_db *db); + +void udma_db_unmap_user(struct udma_dev *udma_dev, struct udma_db *db); + +#endif /* _UDMA_DB_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_device.h b/drivers/ub/hw/hns3/hns3_udma_device.h index 6bea23607708..dec48c5cc128 100644 --- a/drivers/ub/hw/hns3/hns3_udma_device.h +++ b/drivers/ub/hw/hns3/hns3_udma_device.h @@ -215,6 +215,13 @@ struct udma_mtr { struct udma_hem_cfg hem_cfg; /* config for hardware addressing */ }; +struct udma_user_db_page { + struct list_head list; + struct ubcore_umem *umem; + uint64_t user_virt; + refcount_t refcount; +}; + struct udma_ceqe { uint32_t comp; uint32_t rsv[15]; @@ -251,6 +258,14 @@ struct udma_work { }; struct udma_dev; + +struct udma_db { + uint32_t *db_record; + struct udma_user_db_page *user_page; + dma_addr_t dma; + void *virt_addr; +}; + struct udma_ucontext { struct ubcore_ucontext uctx; struct udma_uar uar; -- Gitee From 59589d96394459537f9df729765fba35570a3a8f Mon Sep 17 00:00:00 2001 From: Weibo Zhao Date: Wed, 16 Aug 2023 16:27:35 +0800 Subject: [PATCH 10/42] hns3 udma: create and destroy jfc driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA ---------------------------------------------------------- JFC(jetty for complete) is a jetty that hardware reports complete state of task. Signed-off-by: Weibo Zhao --- drivers/ub/hw/hns3/Makefile | 2 +- drivers/ub/hw/hns3/hns3_udma_abi.h | 14 + drivers/ub/hw/hns3/hns3_udma_common.h | 1 + drivers/ub/hw/hns3/hns3_udma_device.h | 45 +++ drivers/ub/hw/hns3/hns3_udma_jfc.c | 430 ++++++++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_jfc.h | 30 ++ drivers/ub/hw/hns3/hns3_udma_main.c | 2 + 7 files changed, 523 insertions(+), 1 deletion(-) create mode 100644 drivers/ub/hw/hns3/hns3_udma_jfc.c diff --git a/drivers/ub/hw/hns3/Makefile b/drivers/ub/hw/hns3/Makefile index 38f7b84cf7e1..804ab976032a 100644 --- a/drivers/ub/hw/hns3/Makefile +++ b/drivers/ub/hw/hns3/Makefile @@ -9,6 +9,6 @@ ccflags-y += -I$(srctree)/drivers/net/ethernet/hisilicon/hns3/ \ $(MODULE_NAME)-objs := hns3_udma_hw.o hns3_udma_main.o hns3_udma_cmd.o \ hns3_udma_hem.o hns3_udma_qp.o hns3_udma_eq.o \ - hns3_udma_db.o + hns3_udma_db.o hns3_udma_jfc.o obj-$(CONFIG_UB_UDMA_HNS3) := hns3_udma.o diff --git a/drivers/ub/hw/hns3/hns3_udma_abi.h b/drivers/ub/hw/hns3/hns3_udma_abi.h index 358a57c78bfd..ae4a1c579d2c 100644 --- a/drivers/ub/hw/hns3/hns3_udma_abi.h +++ b/drivers/ub/hw/hns3/hns3_udma_abi.h @@ -19,11 +19,25 @@ #include #define MAP_COMMAND_MASK 0xff +#define UDMA_ADDR_4K_MASK 0xfffUL enum { UDMA_MMAP_UAR_PAGE, }; +struct udma_create_jfc_ucmd { + uint64_t buf_addr; + uint64_t db_addr; +}; + +enum udma_jfc_cap_flags { + UDMA_JFC_CAP_RECORD_DB = 1 << 0, +}; + +struct udma_create_jfc_resp { + uint32_t jfc_caps; +}; + struct udma_create_ctx_resp { uint32_t num_comp_vectors; uint32_t num_qps_shift; diff --git a/drivers/ub/hw/hns3/hns3_udma_common.h b/drivers/ub/hw/hns3/hns3_udma_common.h index a092f66e7f33..f9448114eca6 100644 --- a/drivers/ub/hw/hns3/hns3_udma_common.h +++ b/drivers/ub/hw/hns3/hns3_udma_common.h @@ -21,6 +21,7 @@ #define ub_read(dev, reg) readl((dev)->reg_base + (reg)) #define UDMA_UDP_DPORT 4791 +#define DMA_DB_RECORD_SHIFT 1 #define udma_get_field(origin, mask, shift) \ ((le32_to_cpu(origin) & (mask)) >> (uint32_t)(shift)) diff --git a/drivers/ub/hw/hns3/hns3_udma_device.h b/drivers/ub/hw/hns3/hns3_udma_device.h index dec48c5cc128..d95033aefbfa 100644 --- a/drivers/ub/hw/hns3/hns3_udma_device.h +++ b/drivers/ub/hw/hns3/hns3_udma_device.h @@ -29,6 +29,9 @@ /* Configure to HW for PAGE_SIZE larger than 4KB */ #define PG_SHIFT_OFFSET (PAGE_SHIFT - 12) + +#define CQ_BANKID_SHIFT 2 + #define UDMA_MAX_IRQ_NUM 128 #define MTT_MIN_COUNT 2 @@ -62,6 +65,42 @@ #define UDMA_MAX_BT_REGION 3 #define UDMA_MAX_BT_LEVEL 3 + +#define CQC_FIELD_LOC(h, l) ((uint64_t)(h) << 32 | (l)) + +#define CQC_CQE_BA_L_OFFSET 3 +#define CQC_CQE_BA_H_OFFSET 35 + +#define CQC_CQ_ST CQC_FIELD_LOC(1, 0) +#define CQC_NOTIFY_MODE CQC_FIELD_LOC(4, 4) +#define CQC_ARM_ST CQC_FIELD_LOC(7, 6) +#define CQC_SHIFT CQC_FIELD_LOC(12, 8) +#define CQC_CQN CQC_FIELD_LOC(55, 32) +#define CQC_POE_EN CQC_FIELD_LOC(56, 56) +#define CQC_POE_NUM CQC_FIELD_LOC(58, 57) +#define CQC_CQE_SIZE CQC_FIELD_LOC(60, 59) +#define CQC_NOTIFY_DEVICE_EN CQC_FIELD_LOC(62, 62) +#define CQC_CQE_CUR_BLK_ADDR_L CQC_FIELD_LOC(95, 64) +#define CQC_CQE_CUR_BLK_ADDR_H CQC_FIELD_LOC(115, 96) +#define CQC_POE_QID CQC_FIELD_LOC(125, 116) +#define CQC_CQE_HOP_NUM CQC_FIELD_LOC(127, 126) +#define CQC_CQE_NEX_BLK_ADDR_L CQC_FIELD_LOC(159, 128) +#define CQC_CQE_NEX_BLK_ADDR_H CQC_FIELD_LOC(179, 160) +#define CQC_CQE_BAR_PG_SZ CQC_FIELD_LOC(187, 184) +#define CQC_CQE_BUF_PG_SZ CQC_FIELD_LOC(191, 188) +#define CQC_CQE_BA_L CQC_FIELD_LOC(287, 256) +#define CQC_CQE_BA_H CQC_FIELD_LOC(316, 288) +#define CQC_DB_RECORD_EN CQC_FIELD_LOC(320, 320) +#define CQC_CQE_DB_RECORD_ADDR_L CQC_FIELD_LOC(351, 321) +#define CQC_CQE_DB_RECORD_ADDR_H CQC_FIELD_LOC(383, 352) +#define CQC_CQE_CNT CQC_FIELD_LOC(407, 384) +#define CQC_CQ_MAX_CNT CQC_FIELD_LOC(431, 416) +#define CQC_CQ_PERIOD CQC_FIELD_LOC(447, 432) +#define CQC_NOTIFY_EN CQC_FIELD_LOC(510, 510) + +#define UDMA_CQ_DEFAULT_BURST_NUM 0x0 +#define UDMA_CQ_DEFAULT_INTERVAL 0x0 + #define EQ_ENABLE 1 #define EQ_DISABLE 0 #define UDMA_CEQ 0 @@ -86,6 +125,12 @@ enum { NO_ARMED = 0x0 }; +enum { + CQE_SIZE_64B = 0x1 +}; + +#define CQ_STATE_VALID 1 + enum udma_reset_stage { UDMA_STATE_RST_DOWN = 2, UDMA_STATE_RST_UNINIT, diff --git a/drivers/ub/hw/hns3/hns3_udma_jfc.c b/drivers/ub/hw/hns3/hns3_udma_jfc.c new file mode 100644 index 000000000000..2fe83aa54e3e --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_jfc.c @@ -0,0 +1,430 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include +#include "hns3_udma_hem.h" +#include "hns3_udma_cmd.h" +#include "hns3_udma_db.h" +#include "hns3_udma_jfc.h" + +static int udma_hw_create_cq(struct udma_dev *dev, + struct udma_cmd_mailbox *mailbox, uint32_t cqn) +{ + struct udma_cmq_desc desc; + struct udma_mbox *mb; + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mbox_desc_init(mb, mailbox->dma, 0, cqn, UDMA_CMD_CREATE_CQC); + + return udma_cmd_mbox(dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); +} + +static int udma_hw_destroy_cq(struct udma_dev *dev, uint32_t cqn) +{ + struct udma_cmq_desc desc; + struct udma_mbox *mb; + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mbox_desc_init(mb, 0, 0, cqn, UDMA_CMD_DESTROY_CQC); + + return udma_cmd_mbox(dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); +} + +static int check_jfc_cfg(struct udma_dev *udma_dev, const struct ubcore_jfc_cfg *cfg) +{ + if (!cfg->depth || cfg->depth > udma_dev->caps.max_cqes) { + dev_err(udma_dev->dev, + "failed to check jfc attr depth = 0x%x.\n", + cfg->depth); + return -EINVAL; + } + + return 0; +} + +static int check_create_jfc(struct udma_dev *udma_dev, + const struct ubcore_jfc_cfg *cfg, + struct udma_create_jfc_ucmd *ucmd, + struct ubcore_udata *udata) +{ + int ret; + + if (udata) { + ret = copy_from_user((void *)ucmd, + (void *)udata->udrv_data->in_addr, + min(udata->udrv_data->in_len, + (uint32_t)sizeof(struct udma_create_jfc_ucmd))); + if (ret) { + dev_err(udma_dev->dev, + "failed to copy JFC udata, ret = %d.\n", ret); + return ret; + } + } + + ret = check_jfc_cfg(udma_dev, cfg); + if (ret) { + dev_err(udma_dev->dev, "failed to check JFC cfg.\n"); + return ret; + } + + return 0; +} + +void set_jfc_param(struct udma_jfc *udma_jfc, const struct ubcore_jfc_cfg *cfg) +{ + udma_jfc->jfc_depth = roundup_pow_of_two(cfg->depth); + memcpy(&udma_jfc->ubcore_jfc.jfc_cfg, cfg, sizeof(struct ubcore_jfc_cfg)); +} + +static void init_jfc(struct udma_jfc *udma_jfc, struct udma_create_jfc_ucmd *ucmd) +{ + spin_lock_init(&udma_jfc->lock); + INIT_LIST_HEAD(&udma_jfc->sq_list); + INIT_LIST_HEAD(&udma_jfc->rq_list); +} + +static int alloc_jfc_cqe_buf(struct udma_dev *dev, struct udma_jfc *jfc, + struct ubcore_udata *udata, uint64_t addr) +{ + struct udma_buf_attr buf_attr = {}; + int ret; + + buf_attr.page_shift = dev->caps.cqe_buf_pg_sz + PAGE_SHIFT; + buf_attr.region[0].size = jfc->jfc_depth * dev->caps.cqe_sz; + buf_attr.region[0].hopnum = dev->caps.cqe_hop_num; + buf_attr.region_count = 1; + ret = udma_mtr_create(dev, &jfc->mtr, &buf_attr, + dev->caps.cqe_ba_pg_sz + PAGE_SHIFT, + addr, udata ? true : false); + if (ret) + dev_err(dev->dev, + "failed to alloc JFC buf, ret = %d.\n", ret); + + return ret; +} + +static void free_jfc_cqe_buf(struct udma_dev *dev, struct udma_jfc *jfc) +{ + udma_mtr_destroy(dev, &jfc->mtr); +} + +static int alloc_jfc_buf(struct udma_dev *udma_dev, struct udma_jfc *udma_jfc, + struct ubcore_udata *udata, + struct udma_create_jfc_ucmd *ucmd) +{ + struct udma_create_jfc_resp resp = {}; + int ret; + + ret = alloc_jfc_cqe_buf(udma_dev, udma_jfc, udata, ucmd->buf_addr); + if (ret) + return ret; + + if (udma_dev->caps.flags & UDMA_CAP_FLAG_CQ_RECORD_DB) { + ret = udma_db_map_user(udma_dev, ucmd->db_addr, &udma_jfc->db); + if (ret) { + dev_err(udma_dev->dev, + "failed to map JFC db, ret = %d.\n", ret); + goto db_err; + } + udma_jfc->jfc_caps |= UDMA_JFC_CAP_RECORD_DB; + } + + if (udata) { + resp.jfc_caps = udma_jfc->jfc_caps; + ret = copy_to_user((void *)udata->udrv_data->out_addr, &resp, + min(udata->udrv_data->out_len, + (uint32_t)sizeof(resp))); + if (ret) { + dev_err(udma_dev->dev, + "failed to copy jfc resp, ret = %d\n", ret); + goto err_copy; + } + } + refcount_set(&udma_jfc->refcount, 1); + init_completion(&udma_jfc->free); + return ret; + +err_copy: + if (udma_dev->caps.flags & UDMA_CAP_FLAG_CQ_RECORD_DB) { + udma_db_unmap_user(udma_dev, &udma_jfc->db); + udma_jfc->jfc_caps &= ~UDMA_JFC_CAP_RECORD_DB; + } +db_err: + free_jfc_cqe_buf(udma_dev, udma_jfc); + + return ret; +} + +static void udma_write_jfc_cqc(struct udma_dev *udma_dev, struct udma_jfc *udma_jfc, + void *mb_buf, uint64_t *mtts, uint64_t dma_handle) +{ + struct udma_jfc_context *jfc_context; + + jfc_context = (struct udma_jfc_context *)mb_buf; + memset(jfc_context, 0, sizeof(*jfc_context)); + + udma_reg_write(jfc_context, CQC_CQ_ST, CQ_STATE_VALID); + udma_reg_write(jfc_context, CQC_ARM_ST, NO_ARMED); + + udma_reg_write(jfc_context, CQC_SHIFT, ilog2(udma_jfc->jfc_depth)); + udma_reg_write(jfc_context, CQC_CQN, udma_jfc->cqn); + + udma_reg_write(jfc_context, CQC_CQE_SIZE, CQE_SIZE_64B); + + udma_reg_write(jfc_context, CQC_CQE_CUR_BLK_ADDR_L, + to_hr_hw_page_addr(mtts[0])); + udma_reg_write(jfc_context, CQC_CQE_CUR_BLK_ADDR_H, + upper_32_bits(to_hr_hw_page_addr(mtts[0]))); + udma_reg_write(jfc_context, CQC_CQE_HOP_NUM, + udma_dev->caps.cqe_hop_num == + UDMA_HOP_NUM_0 ? 0 : udma_dev->caps.cqe_hop_num); + udma_reg_write(jfc_context, CQC_CQE_NEX_BLK_ADDR_L, + to_hr_hw_page_addr(mtts[1])); + udma_reg_write(jfc_context, CQC_CQE_NEX_BLK_ADDR_H, + upper_32_bits(to_hr_hw_page_addr(mtts[1]))); + udma_reg_write(jfc_context, CQC_CQE_BAR_PG_SZ, + to_hr_hw_page_shift(udma_jfc->mtr.hem_cfg.ba_pg_shift)); + udma_reg_write(jfc_context, CQC_CQE_BUF_PG_SZ, + to_hr_hw_page_shift(udma_jfc->mtr.hem_cfg.buf_pg_shift)); + udma_reg_write(jfc_context, CQC_CQE_BA_L, + dma_handle >> CQC_CQE_BA_L_OFFSET); + udma_reg_write(jfc_context, CQC_CQE_BA_H, + dma_handle >> CQC_CQE_BA_H_OFFSET); + udma_reg_write(jfc_context, CQC_CQ_MAX_CNT, UDMA_CQ_DEFAULT_BURST_NUM); + udma_reg_write(jfc_context, CQC_CQ_PERIOD, UDMA_CQ_DEFAULT_INTERVAL); + if (udma_jfc->jfc_caps & UDMA_JFC_CAP_RECORD_DB) { + udma_reg_enable(jfc_context, CQC_DB_RECORD_EN); + udma_reg_write(jfc_context, CQC_CQE_DB_RECORD_ADDR_L, + lower_32_bits(udma_jfc->db.dma) >> + DMA_DB_RECORD_SHIFT); + udma_reg_write(jfc_context, CQC_CQE_DB_RECORD_ADDR_H, + upper_32_bits(udma_jfc->db.dma)); + } +} + +static int udma_create_jfc_cqc(struct udma_dev *udma_dev, + struct udma_jfc *udma_jfc, + uint64_t *mtts, uint64_t dma_handle) +{ + struct udma_cmd_mailbox *mailbox; + int ret; + + mailbox = udma_alloc_cmd_mailbox(udma_dev); + if (IS_ERR(mailbox)) { + dev_err(udma_dev->dev, "failed to alloc mailbox for CQC.\n"); + return PTR_ERR(mailbox); + } + + udma_write_jfc_cqc(udma_dev, udma_jfc, mailbox->buf, mtts, dma_handle); + + ret = udma_hw_create_cq(udma_dev, mailbox, udma_jfc->cqn); + if (ret) + dev_err(udma_dev->dev, + "failed to send create cmd for jfc(0x%llx), ret = %d.\n", + udma_jfc->cqn, ret); + + udma_free_cmd_mailbox(udma_dev, mailbox); + + return ret; +} + +static int alloc_jfc_cqc(struct udma_dev *udma_dev, struct udma_jfc *udma_jfc) +{ + struct udma_jfc_table *jfc_table = &udma_dev->jfc_table; + uint64_t mtts[MTT_MIN_COUNT] = {}; + uint64_t dma_handle; + int ret; + + ret = udma_mtr_find(udma_dev, &udma_jfc->mtr, 0, mtts, + ARRAY_SIZE(mtts), &dma_handle); + if (!ret) { + dev_err(udma_dev->dev, + "failed to find JFC mtr, ret = %d.\n", ret); + return -EINVAL; + } + + ret = udma_table_get(udma_dev, &jfc_table->table, udma_jfc->cqn); + if (ret) { + dev_err(udma_dev->dev, + "failed to get JFC(0x%llx) context, ret = %d.\n", + udma_jfc->cqn, ret); + return ret; + } + + ret = xa_err(xa_store(&jfc_table->xa, udma_jfc->cqn, udma_jfc, + GFP_KERNEL)); + if (ret) { + dev_err(udma_dev->dev, "failed to store JFC, ret = %d.\n", ret); + goto err_put; + } + + ret = udma_create_jfc_cqc(udma_dev, udma_jfc, mtts, dma_handle); + if (ret) + goto err_xa; + + udma_jfc->arm_sn = 1; + return 0; + +err_xa: + xa_erase(&jfc_table->xa, udma_jfc->cqn); +err_put: + udma_table_put(udma_dev, &jfc_table->table, udma_jfc->cqn); + + return ret; +} + +static void free_jfc_cqc(struct udma_dev *udma_dev, struct udma_jfc *udma_jfc) +{ + struct udma_jfc_table *jfc_table = &udma_dev->jfc_table; + int ret; + + ret = udma_hw_destroy_cq(udma_dev, udma_jfc->cqn); + if (ret) + dev_err(udma_dev->dev, "destroy failed (%d) for JFC %06llx\n", + ret, udma_jfc->cqn); + + xa_erase(&jfc_table->xa, udma_jfc->cqn); + udma_table_put(udma_dev, &jfc_table->table, udma_jfc->cqn); +} + +static void free_jfc_buf(struct udma_dev *udma_dev, struct udma_jfc *udma_jfc) +{ + /* wait for all interrupt processed */ + if (refcount_dec_and_test(&udma_jfc->refcount)) + complete(&udma_jfc->free); + wait_for_completion(&udma_jfc->free); + + if (udma_dev->caps.flags & UDMA_CAP_FLAG_CQ_RECORD_DB) + udma_db_unmap_user(udma_dev, &udma_jfc->db); + udma_mtr_destroy(udma_dev, &udma_jfc->mtr); +} + +static uint8_t get_least_load_bankid_for_jfc(struct udma_bank *bank) +{ + uint32_t least_load = bank[0].inuse; + uint8_t bankid = 0; + uint32_t bankcnt; + uint8_t i; + + for (i = 1; i < UDMA_CQ_BANK_NUM; i++) { + bankcnt = bank[i].inuse; + if (bankcnt < least_load) { + least_load = bankcnt; + bankid = i; + } + } + + return bankid; +} + +static int alloc_jfc_id(struct udma_dev *udma_dev, struct udma_jfc *udma_jfc) +{ + struct udma_jfc_table *jfc_table = &udma_dev->jfc_table; + struct udma_bank *bank; + uint8_t bankid; + int id; + + mutex_lock(&jfc_table->bank_mutex); + bankid = get_least_load_bankid_for_jfc(jfc_table->bank); + bank = &jfc_table->bank[bankid]; + + id = ida_alloc_range(&bank->ida, bank->min, bank->max, GFP_KERNEL); + if (id < 0) { + mutex_unlock(&jfc_table->bank_mutex); + return id; + } + + /* the lower 2 bits is bankid */ + udma_jfc->cqn = (id << CQ_BANKID_SHIFT) | bankid; + bank->inuse++; + mutex_unlock(&jfc_table->bank_mutex); + udma_jfc->ubcore_jfc.id = udma_jfc->cqn; + + return 0; +} + +static void free_jfc_id(struct udma_dev *udma_dev, struct udma_jfc *udma_jfc) +{ + struct udma_jfc_table *jfc_table = &udma_dev->jfc_table; + struct udma_bank *bank; + + bank = &jfc_table->bank[get_jfc_bankid(udma_jfc->cqn)]; + ida_free(&bank->ida, udma_jfc->cqn >> CQ_BANKID_SHIFT); + + mutex_lock(&jfc_table->bank_mutex); + bank->inuse--; + mutex_unlock(&jfc_table->bank_mutex); +} + +struct ubcore_jfc *udma_create_jfc(struct ubcore_device *dev, const struct ubcore_jfc_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + struct udma_create_jfc_ucmd ucmd = {}; + struct udma_jfc *udma_jfc; + int ret; + + ret = check_create_jfc(udma_dev, cfg, &ucmd, udata); + if (ret) + goto err; + + udma_jfc = kzalloc(sizeof(*udma_jfc), GFP_KERNEL); + if (ZERO_OR_NULL_PTR(udma_jfc)) { + ret = -ENOMEM; + goto err; + } + + init_jfc(udma_jfc, &ucmd); + + set_jfc_param(udma_jfc, cfg); + + ret = alloc_jfc_buf(udma_dev, udma_jfc, udata, &ucmd); + if (ret) + goto err_jfc; + + ret = alloc_jfc_id(udma_dev, udma_jfc); + if (ret) + goto err_jfc_buf; + + ret = alloc_jfc_cqc(udma_dev, udma_jfc); + if (ret) + goto err_jfc_id; + + return &udma_jfc->ubcore_jfc; + +err_jfc_id: + free_jfc_id(udma_dev, udma_jfc); +err_jfc_buf: + free_jfc_buf(udma_dev, udma_jfc); +err_jfc: + kfree(udma_jfc); +err: + return NULL; +} + +int udma_destroy_jfc(struct ubcore_jfc *jfc) +{ + struct udma_dev *udma_dev = to_udma_dev(jfc->ub_dev); + struct udma_jfc *udma_jfc = to_udma_jfc(jfc); + + free_jfc_cqc(udma_dev, udma_jfc); + free_jfc_id(udma_dev, udma_jfc); + free_jfc_buf(udma_dev, udma_jfc); + kfree(udma_jfc); + + return 0; +} diff --git a/drivers/ub/hw/hns3/hns3_udma_jfc.h b/drivers/ub/hw/hns3/hns3_udma_jfc.h index dc4961830a31..3844311715f3 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jfc.h +++ b/drivers/ub/hw/hns3/hns3_udma_jfc.h @@ -17,6 +17,36 @@ #define _UDMA_JFC_H #include "hns3_udma_device.h" +#include "hns3_udma_abi.h" + +struct udma_jfc { + struct ubcore_jfc ubcore_jfc; + struct udma_mtr mtr; + uint32_t jfc_caps; + uint32_t jfc_depth; + uint64_t cqn; + uint32_t arm_sn; + struct udma_db db; + spinlock_t lock; + refcount_t refcount; + struct completion free; + struct list_head sq_list; + struct list_head rq_list; +}; + +#define UDMA_JFC_CONTEXT_SIZE 16 +struct udma_jfc_context { + uint32_t jfc_data[UDMA_JFC_CONTEXT_SIZE]; +}; + +static inline struct udma_jfc *to_udma_jfc(struct ubcore_jfc *jfc) +{ + return container_of(jfc, struct udma_jfc, ubcore_jfc); +} + +struct ubcore_jfc *udma_create_jfc(struct ubcore_device *dev, const struct ubcore_jfc_cfg *cfg, + struct ubcore_udata *udata); +int udma_destroy_jfc(struct ubcore_jfc *jfc); static inline uint8_t get_jfc_bankid(uint64_t cqn) { /* The lower 2 bits of CQN are used to hash to different banks */ diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c index 34e87b1e8864..40caf0d42535 100644 --- a/drivers/ub/hw/hns3/hns3_udma_main.c +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -166,6 +166,8 @@ static struct ubcore_ops g_udma_dev_ops = { .alloc_ucontext = udma_alloc_ucontext, .free_ucontext = udma_free_ucontext, .mmap = udma_mmap, + .create_jfc = udma_create_jfc, + .destroy_jfc = udma_destroy_jfc, }; static void udma_cleanup_uar_table(struct udma_dev *dev) -- Gitee From f6d3d88161e04c9d3a5519d44afb3b79ea31ee96 Mon Sep 17 00:00:00 2001 From: Weibo Zhao Date: Thu, 17 Aug 2023 14:58:44 +0800 Subject: [PATCH 11/42] hns3 udma: support of notify address in create jfc driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA ------------------------------------------------- Notify address is an address used by write_with_notify. Write_with_notify can write a data to notify_address after write operation. Signed-off-by: Weibo Zhao --- drivers/ub/hw/hns3/hns3_udma_abi.h | 23 ++++++ drivers/ub/hw/hns3/hns3_udma_device.h | 6 ++ drivers/ub/hw/hns3/hns3_udma_jfc.c | 103 ++++++++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_jfc.h | 23 ++++++ 4 files changed, 155 insertions(+) diff --git a/drivers/ub/hw/hns3/hns3_udma_abi.h b/drivers/ub/hw/hns3/hns3_udma_abi.h index ae4a1c579d2c..77c1461b9d8d 100644 --- a/drivers/ub/hw/hns3/hns3_udma_abi.h +++ b/drivers/ub/hw/hns3/hns3_udma_abi.h @@ -25,9 +25,32 @@ enum { UDMA_MMAP_UAR_PAGE, }; +enum udma_jfc_init_attr_mask { + UDMA_JFC_NOTIFY_CREATE_FLAGS = 1 << 0, +}; + +enum udma_jfc_create_flags { + UDMA_JFC_CREATE_ENABLE_NOTIFY = 1 << 1, +}; + +enum udma_jfc_notify_mode { + UDMA_JFC_NOTIFY_MODE_64B_ALIGN, + UDMA_JFC_NOTIFY_MODE_4B_ALIGN, + UDMA_JFC_NOTIFY_MODE_DDR_64B_ALIGN, + UDMA_JFC_NOTIFY_MODE_DDR_4B_ALIGN, +}; + +struct udma_jfc_attr_ex { + uint64_t jfc_ex_mask; /* Use enum udma_jfc_init_attr_mask */ + uint64_t create_flags; /* Use enum udma_jfc_create_flags */ + uint64_t notify_addr; + uint8_t notify_mode; /* Use enum udma_jfc_notify_mode */ +}; + struct udma_create_jfc_ucmd { uint64_t buf_addr; uint64_t db_addr; + struct udma_jfc_attr_ex jfc_attr_ex; }; enum udma_jfc_cap_flags { diff --git a/drivers/ub/hw/hns3/hns3_udma_device.h b/drivers/ub/hw/hns3/hns3_udma_device.h index d95033aefbfa..af5f4970048e 100644 --- a/drivers/ub/hw/hns3/hns3_udma_device.h +++ b/drivers/ub/hw/hns3/hns3_udma_device.h @@ -75,6 +75,7 @@ #define CQC_NOTIFY_MODE CQC_FIELD_LOC(4, 4) #define CQC_ARM_ST CQC_FIELD_LOC(7, 6) #define CQC_SHIFT CQC_FIELD_LOC(12, 8) +#define CQC_NOTIFY_ADDR_0 CQC_FIELD_LOC(31, 24) #define CQC_CQN CQC_FIELD_LOC(55, 32) #define CQC_POE_EN CQC_FIELD_LOC(56, 56) #define CQC_POE_NUM CQC_FIELD_LOC(58, 57) @@ -86,16 +87,21 @@ #define CQC_CQE_HOP_NUM CQC_FIELD_LOC(127, 126) #define CQC_CQE_NEX_BLK_ADDR_L CQC_FIELD_LOC(159, 128) #define CQC_CQE_NEX_BLK_ADDR_H CQC_FIELD_LOC(179, 160) +#define CQC_NOTIFY_ADDR_2 CQC_FIELD_LOC(183, 180) #define CQC_CQE_BAR_PG_SZ CQC_FIELD_LOC(187, 184) #define CQC_CQE_BUF_PG_SZ CQC_FIELD_LOC(191, 188) +#define CQC_NOTIFY_ADDR_3 CQC_FIELD_LOC(223, 216) +#define CQC_NOTIFY_ADDR_4 CQC_FIELD_LOC(255, 248) #define CQC_CQE_BA_L CQC_FIELD_LOC(287, 256) #define CQC_CQE_BA_H CQC_FIELD_LOC(316, 288) #define CQC_DB_RECORD_EN CQC_FIELD_LOC(320, 320) #define CQC_CQE_DB_RECORD_ADDR_L CQC_FIELD_LOC(351, 321) #define CQC_CQE_DB_RECORD_ADDR_H CQC_FIELD_LOC(383, 352) #define CQC_CQE_CNT CQC_FIELD_LOC(407, 384) +#define CQC_NOTIFY_ADDR_5 CQC_FIELD_LOC(415, 408) #define CQC_CQ_MAX_CNT CQC_FIELD_LOC(431, 416) #define CQC_CQ_PERIOD CQC_FIELD_LOC(447, 432) +#define CQC_NOTIFY_ADDR_6 CQC_FIELD_LOC(509, 504) #define CQC_NOTIFY_EN CQC_FIELD_LOC(510, 510) #define UDMA_CQ_DEFAULT_BURST_NUM 0x0 diff --git a/drivers/ub/hw/hns3/hns3_udma_jfc.c b/drivers/ub/hw/hns3/hns3_udma_jfc.c index 2fe83aa54e3e..301d2baf6bbf 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jfc.c +++ b/drivers/ub/hw/hns3/hns3_udma_jfc.c @@ -56,6 +56,56 @@ static int check_jfc_cfg(struct udma_dev *udma_dev, const struct ubcore_jfc_cfg return 0; } +static int check_notify_attr(struct udma_dev *udma_dev, + struct udma_jfc_attr_ex *jfc_attr_ex) +{ + if (!(udma_dev->caps.flags & UDMA_CAP_FLAG_WRITE_NOTIFY)) { + dev_err(udma_dev->dev, "Unsupport NOTIFY JFC.\n"); + return -EINVAL; + } + + switch (jfc_attr_ex->notify_mode) { + case UDMA_JFC_NOTIFY_MODE_4B_ALIGN: + case UDMA_JFC_NOTIFY_MODE_DDR_4B_ALIGN: + break; + case UDMA_JFC_NOTIFY_MODE_64B_ALIGN: + case UDMA_JFC_NOTIFY_MODE_DDR_64B_ALIGN: + dev_err(udma_dev->dev, "Doesn't support notify mode %u\n", + jfc_attr_ex->notify_mode); + return -EINVAL; + default: + dev_err(udma_dev->dev, "Invalid notify mode %u\n", + jfc_attr_ex->notify_mode); + return -EINVAL; + } + + if (jfc_attr_ex->notify_addr & UDMA_ADDR_4K_MASK) { + dev_err(udma_dev->dev, + "Notify addr should be aligned to 4k.\n"); + return -EINVAL; + } + + return 0; +} + +static int check_jfc_attr_ex(struct udma_dev *udma_dev, + struct udma_jfc_attr_ex *jfc_attr_ex) +{ + int ret; + + switch (jfc_attr_ex->create_flags) { + case UDMA_JFC_CREATE_ENABLE_NOTIFY: + ret = check_notify_attr(udma_dev, jfc_attr_ex); + break; + default: + dev_err(udma_dev->dev, "Invalid create flags %llu\n", + jfc_attr_ex->create_flags); + return -EINVAL; + } + + return ret; +} + static int check_create_jfc(struct udma_dev *udma_dev, const struct ubcore_jfc_cfg *cfg, struct udma_create_jfc_ucmd *ucmd, @@ -81,6 +131,16 @@ static int check_create_jfc(struct udma_dev *udma_dev, return ret; } + if (ucmd->jfc_attr_ex.jfc_ex_mask & + UDMA_JFC_NOTIFY_CREATE_FLAGS) { + ret = check_jfc_attr_ex(udma_dev, &ucmd->jfc_attr_ex); + if (ret) { + dev_err(udma_dev->dev, + "failed to check JFC attr ex.\n"); + return ret; + } + } + return 0; } @@ -95,6 +155,8 @@ static void init_jfc(struct udma_jfc *udma_jfc, struct udma_create_jfc_ucmd *ucm spin_lock_init(&udma_jfc->lock); INIT_LIST_HEAD(&udma_jfc->sq_list); INIT_LIST_HEAD(&udma_jfc->rq_list); + if (ucmd->jfc_attr_ex.jfc_ex_mask & UDMA_JFC_NOTIFY_CREATE_FLAGS) + udma_jfc->jfc_attr_ex = ucmd->jfc_attr_ex; } static int alloc_jfc_cqe_buf(struct udma_dev *dev, struct udma_jfc *jfc, @@ -169,6 +231,44 @@ static int alloc_jfc_buf(struct udma_dev *udma_dev, struct udma_jfc *udma_jfc, return ret; } +static void set_write_notify_param(struct udma_dev *udma_dev, + struct udma_jfc *udma_jfc, + struct udma_jfc_context *jfc_context) +{ + uint8_t device_mode; + + if (udma_jfc->jfc_attr_ex.notify_mode == UDMA_JFC_NOTIFY_MODE_4B_ALIGN) + device_mode = UDMA_NOTIFY_DEV; + else + device_mode = UDMA_NOTIFY_DDR; + + udma_reg_enable(jfc_context, CQC_NOTIFY_EN); + udma_reg_write(jfc_context, CQC_NOTIFY_DEVICE_EN, device_mode); + /* Supports only 4B alignment. */ + udma_reg_write(jfc_context, CQC_NOTIFY_MODE, UDMA_NOTIFY_MODE_4B); + udma_reg_write(jfc_context, CQC_NOTIFY_ADDR_0, + (uint32_t)udma_get_field64(udma_jfc->jfc_attr_ex.notify_addr, + CQC_NOTIFY_ADDR_0_M, CQC_NOTIFY_ADDR_0_S)); + udma_reg_write(jfc_context, CQC_POE_QID, + (uint32_t)udma_get_field64(udma_jfc->jfc_attr_ex.notify_addr, + CQC_NOTIFY_ADDR_1_M, CQC_NOTIFY_ADDR_1_S)); + udma_reg_write(jfc_context, CQC_NOTIFY_ADDR_2, + (uint32_t)udma_get_field64(udma_jfc->jfc_attr_ex.notify_addr, + CQC_NOTIFY_ADDR_2_M, CQC_NOTIFY_ADDR_2_S)); + udma_reg_write(jfc_context, CQC_NOTIFY_ADDR_3, + (uint32_t)udma_get_field64(udma_jfc->jfc_attr_ex.notify_addr, + CQC_NOTIFY_ADDR_3_M, CQC_NOTIFY_ADDR_3_S)); + udma_reg_write(jfc_context, CQC_NOTIFY_ADDR_4, + (uint32_t)udma_get_field64(udma_jfc->jfc_attr_ex.notify_addr, + CQC_NOTIFY_ADDR_4_M, CQC_NOTIFY_ADDR_4_S)); + udma_reg_write(jfc_context, CQC_NOTIFY_ADDR_5, + (uint32_t)udma_get_field64(udma_jfc->jfc_attr_ex.notify_addr, + CQC_NOTIFY_ADDR_5_M, CQC_NOTIFY_ADDR_5_S)); + udma_reg_write(jfc_context, CQC_NOTIFY_ADDR_6, + (uint32_t)udma_get_field64(udma_jfc->jfc_attr_ex.notify_addr, + CQC_NOTIFY_ADDR_6_M, CQC_NOTIFY_ADDR_6_S)); +} + static void udma_write_jfc_cqc(struct udma_dev *udma_dev, struct udma_jfc *udma_jfc, void *mb_buf, uint64_t *mtts, uint64_t dma_handle) { @@ -214,6 +314,9 @@ static void udma_write_jfc_cqc(struct udma_dev *udma_dev, struct udma_jfc *udma_ udma_reg_write(jfc_context, CQC_CQE_DB_RECORD_ADDR_H, upper_32_bits(udma_jfc->db.dma)); } + + if (udma_jfc->jfc_attr_ex.create_flags == UDMA_JFC_CREATE_ENABLE_NOTIFY) + set_write_notify_param(udma_dev, udma_jfc, jfc_context); } static int udma_create_jfc_cqc(struct udma_dev *udma_dev, diff --git a/drivers/ub/hw/hns3/hns3_udma_jfc.h b/drivers/ub/hw/hns3/hns3_udma_jfc.h index 3844311715f3..572b095cedbb 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jfc.h +++ b/drivers/ub/hw/hns3/hns3_udma_jfc.h @@ -32,6 +32,7 @@ struct udma_jfc { struct completion free; struct list_head sq_list; struct list_head rq_list; + struct udma_jfc_attr_ex jfc_attr_ex; }; #define UDMA_JFC_CONTEXT_SIZE 16 @@ -39,6 +40,28 @@ struct udma_jfc_context { uint32_t jfc_data[UDMA_JFC_CONTEXT_SIZE]; }; +#define UDMA_NOTIFY_MODE_4B 1UL + +enum udma_notify_device_en { + UDMA_NOTIFY_DEV, + UDMA_NOTIFY_DDR, +}; + +#define CQC_NOTIFY_ADDR_0_S 12 +#define CQC_NOTIFY_ADDR_0_M GENMASK(19, 12) +#define CQC_NOTIFY_ADDR_1_S 20 +#define CQC_NOTIFY_ADDR_1_M GENMASK(29, 20) +#define CQC_NOTIFY_ADDR_2_S 30 +#define CQC_NOTIFY_ADDR_2_M GENMASK(33, 30) +#define CQC_NOTIFY_ADDR_3_S 34 +#define CQC_NOTIFY_ADDR_3_M GENMASK(41, 34) +#define CQC_NOTIFY_ADDR_4_S 42 +#define CQC_NOTIFY_ADDR_4_M GENMASK(49, 42) +#define CQC_NOTIFY_ADDR_5_S 50 +#define CQC_NOTIFY_ADDR_5_M GENMASK(57, 50) +#define CQC_NOTIFY_ADDR_6_S 58 +#define CQC_NOTIFY_ADDR_6_M GENMASK(63, 58) + static inline struct udma_jfc *to_udma_jfc(struct ubcore_jfc *jfc) { return container_of(jfc, struct udma_jfc, ubcore_jfc); -- Gitee From 8f14e4c3ca68e89bc63fbb12005358ed7e802e60 Mon Sep 17 00:00:00 2001 From: Weibo Zhao Date: Thu, 17 Aug 2023 16:10:59 +0800 Subject: [PATCH 12/42] hns3 udma: support of modify jfc driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA ------------------------------------------------------- This patch add support of modify jfc. This function can modify the aggregation conditions for reporting events. Event can be reported if number reaching MODERATE_COUNT or time reaching MODERATE_PERIOD. Signed-off-by: Weibo Zhao --- drivers/ub/hw/hns3/hns3_udma_jfc.c | 61 +++++++++++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_jfc.h | 2 + drivers/ub/hw/hns3/hns3_udma_main.c | 1 + 3 files changed, 64 insertions(+) diff --git a/drivers/ub/hw/hns3/hns3_udma_jfc.c b/drivers/ub/hw/hns3/hns3_udma_jfc.c index 301d2baf6bbf..e1b3aa3009d6 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jfc.c +++ b/drivers/ub/hw/hns3/hns3_udma_jfc.c @@ -531,3 +531,64 @@ int udma_destroy_jfc(struct ubcore_jfc *jfc) return 0; } + +int udma_modify_jfc(struct ubcore_jfc *ubcore_jfc, const struct ubcore_jfc_attr *attr, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_device = to_udma_dev(ubcore_jfc->ub_dev); + uint16_t cq_period = attr->moderate_period; + uint16_t cq_count = attr->moderate_count; + struct udma_jfc_context *jfc_context; + struct udma_jfc_context *cqc_mask; + struct udma_cmd_mailbox *mailbox; + struct udma_cmq_desc desc; + struct udma_jfc *udma_jfc; + struct udma_mbox *mb; + int ret; + + udma_jfc = to_udma_jfc(ubcore_jfc); + ret = check_jfc_cfg(udma_device, &ubcore_jfc->jfc_cfg); + if (ret) + return ret; + + if (!(attr->mask & (UBCORE_JFC_MODERATE_COUNT | + UBCORE_JFC_MODERATE_PERIOD))) { + dev_err(udma_device->dev, + "JFC modify mask is not set or invalid.\n"); + return -EINVAL; + } + + mailbox = udma_alloc_cmd_mailbox(udma_device); + if (IS_ERR(mailbox)) { + dev_err(udma_device->dev, "failed to alloc mailbox for CQ.\n"); + return -ENOMEM; + } + + jfc_context = (struct udma_jfc_context *)mailbox->buf; + cqc_mask = (struct udma_jfc_context *)mailbox->buf + 1; + + memset(cqc_mask, 0xff, sizeof(*cqc_mask)); + if (attr->mask & UBCORE_JFC_MODERATE_COUNT) { + udma_reg_write(jfc_context, CQC_CQ_MAX_CNT, cq_count); + udma_reg_clear(cqc_mask, CQC_CQ_MAX_CNT); + } + + if (attr->mask & UBCORE_JFC_MODERATE_PERIOD) { + udma_reg_write(jfc_context, CQC_CQ_PERIOD, cq_period); + udma_reg_clear(cqc_mask, CQC_CQ_PERIOD); + } + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mbox_desc_init(mb, mailbox->dma, 0, ubcore_jfc->id, + UDMA_CMD_MODIFY_CQC); + + ret = udma_cmd_mbox(udma_device, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); + if (ret) + dev_err(udma_device->dev, + "failed to send modify cmd for jfc(0x%x), ret = %d.\n", + ubcore_jfc->id, ret); + udma_free_cmd_mailbox(udma_device, mailbox); + + return ret; +} diff --git a/drivers/ub/hw/hns3/hns3_udma_jfc.h b/drivers/ub/hw/hns3/hns3_udma_jfc.h index 572b095cedbb..ddfd56575141 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jfc.h +++ b/drivers/ub/hw/hns3/hns3_udma_jfc.h @@ -70,6 +70,8 @@ static inline struct udma_jfc *to_udma_jfc(struct ubcore_jfc *jfc) struct ubcore_jfc *udma_create_jfc(struct ubcore_device *dev, const struct ubcore_jfc_cfg *cfg, struct ubcore_udata *udata); int udma_destroy_jfc(struct ubcore_jfc *jfc); +int udma_modify_jfc(struct ubcore_jfc *ubcore_jfc, const struct ubcore_jfc_attr *attr, + struct ubcore_udata *udata); static inline uint8_t get_jfc_bankid(uint64_t cqn) { /* The lower 2 bits of CQN are used to hash to different banks */ diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c index 40caf0d42535..73bfd6121ed9 100644 --- a/drivers/ub/hw/hns3/hns3_udma_main.c +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -167,6 +167,7 @@ static struct ubcore_ops g_udma_dev_ops = { .free_ucontext = udma_free_ucontext, .mmap = udma_mmap, .create_jfc = udma_create_jfc, + .modify_jfc = udma_modify_jfc, .destroy_jfc = udma_destroy_jfc, }; -- Gitee From 992594ce6eae868146090d83db39cb9de995234b Mon Sep 17 00:00:00 2001 From: Weibo Zhao Date: Fri, 18 Aug 2023 10:39:01 +0800 Subject: [PATCH 13/42] hns3 udma: support of set eid driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA ------------------------------------------------------------ This patch add support of setting EID(Entity ID). EID will be set in gmv(GUID-MAC-VALN) table. Signed-off-by: Weibo Zhao --- drivers/ub/hw/hns3/hns3_udma_device.h | 1 + drivers/ub/hw/hns3/hns3_udma_hw.c | 83 +++++++++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_hw.h | 56 ++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_main.c | 12 ++++ 4 files changed, 152 insertions(+) diff --git a/drivers/ub/hw/hns3/hns3_udma_device.h b/drivers/ub/hw/hns3/hns3_udma_device.h index af5f4970048e..cab9075caf22 100644 --- a/drivers/ub/hw/hns3/hns3_udma_device.h +++ b/drivers/ub/hw/hns3/hns3_udma_device.h @@ -430,6 +430,7 @@ struct udma_hw { int (*clear_hem)(struct udma_dev *udma_dev, struct udma_hem_table *table, int obj, int step_idx); + int (*set_eid)(struct udma_dev *udma_dev, union ubcore_eid eid); int (*init_eq)(struct udma_dev *udma_dev); void (*cleanup_eq)(struct udma_dev *udma_dev); }; diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.c b/drivers/ub/hw/hns3/hns3_udma_hw.c index 936777ec26e6..f91c6f647848 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hw.c +++ b/drivers/ub/hw/hns3/hns3_udma_hw.c @@ -1006,6 +1006,88 @@ static void udma_cmq_exit(struct udma_dev *udma_dev) udma_free_cmq_desc(udma_dev, &priv->cmq.csq); } +static int config_gmv_table(struct udma_dev *udma_dev, union ubcore_eid eid) +{ + uint32_t sgid_type = SGID_TYPE_IPV4; + struct udma_cfg_gmv_tb_a *tb_a; + struct udma_cfg_gmv_tb_b *tb_b; + struct udma_cmq_desc desc[2]; + uint16_t smac_l; + + tb_a = (struct udma_cfg_gmv_tb_a *)desc[0].data; + tb_b = (struct udma_cfg_gmv_tb_b *)desc[1].data; + + udma_cmq_setup_basic_desc(&desc[0], UDMA_OPC_CFG_GMV_TBL, false); + desc[0].flag |= cpu_to_le16(UDMA_CMD_FLAG_NEXT); + udma_cmq_setup_basic_desc(&desc[1], UDMA_OPC_CFG_GMV_TBL, false); + + smac_l = + *(uint16_t *)&udma_dev->uboe.netdevs[0]->dev_addr[SMAC_L_SHIFT]; + udma_set_field(tb_a->vf_type_vlan_smac, CFG_GMV_TB_VF_SMAC_L_M, + CFG_GMV_TB_VF_SMAC_L_S, smac_l); + tb_a->vf_smac_h = + *(uint32_t *)&udma_dev->uboe.netdevs[0]->dev_addr[SMAC_H_SHIFT]; + udma_set_field(tb_a->vf_type_vlan_smac, CFG_GMV_TB_VF_SGID_TYPE_M, + CFG_GMV_TB_VF_SGID_TYPE_S, sgid_type); + memcpy(tb_a, &eid, sizeof(eid)); + udma_set_bit(tb_a->vf_type_vlan_smac, CFG_GMV_TB_VF_PATTERN_S, 0); + tb_b->vf_id = 0; + + return udma_cmq_send(udma_dev, desc, CFG_GMV_TBL_CMD_NUM); +} + +static void udma_fill_eid_addr(struct udma_eid_tbl_entry_cmd *eid_entry, + union ubcore_eid eid) +{ + int i; + + /* big endian */ + for (i = 0; i < UDMA_EID_SIZE_IDX; i++) + eid_entry->eid_addr[i] = (*(((uint32_t *)eid.raw) + i)); +} + +static int set_eid_table(struct udma_dev *udma_dev, union ubcore_eid eid) +{ + struct udma_eid_tbl_entry_cmd *eid_entry; + struct udma_cmq_desc desc = {}; + uint8_t resp_code; + int ret; + + eid_entry = (struct udma_eid_tbl_entry_cmd *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_DEID_TBL_ADD, false); + udma_set_field(eid_entry->eid_ad, UDMA_EID_TB_VFID_M, + UDMA_EID_TB_VFID_S, 0); + udma_fill_eid_addr(eid_entry, eid); + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) { + dev_err(udma_dev->dev, "Send set eid table cmd failed.\n"); + return ret; + } + + resp_code = (le32_to_cpu(desc.data[0])) & 0xff; + if (resp_code == UDMA_EID_TB_RES_SUCCESS || + resp_code == UDMA_EID_TB_RES_MODIFY) + return 0; + else + return -EIO; +} + +static int udma_hw_set_eid(struct udma_dev *udma_dev, union ubcore_eid eid) +{ + int ret; + + ret = config_gmv_table(udma_dev, eid); + if (ret) { + dev_err(udma_dev->dev, "Set EID to GMV table failed.\n"); + return ret; + } + + ret = set_eid_table(udma_dev, eid); + if (ret) + dev_err(udma_dev->dev, "Set EID table failed.\n"); + + return ret; +} static void config_llm_table(struct udma_buf *data_buf, void *cfg_buf) { @@ -1456,6 +1538,7 @@ static const struct udma_hw udma_hw = { .poll_mbox_done = udma_poll_mbox_done, .set_hem = udma_set_hem, .clear_hem = udma_clear_hem, + .set_eid = udma_hw_set_eid, .init_eq = udma_init_eq_table, .cleanup_eq = udma_cleanup_eq_table, }; diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.h b/drivers/ub/hw/hns3/hns3_udma_hw.h index 250e78631c86..f36116e22b18 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hw.h +++ b/drivers/ub/hw/hns3/hns3_udma_hw.h @@ -202,6 +202,8 @@ enum udma_opcode_type { UDMA_OPC_EXT_CFG = 0x8512, UDMA_SWITCH_PARAMETER_CFG = 0x1033, UDMA_QUERY_OOR_CAPS = 0xA002, + UDMA_OPC_DEID_TBL_ADD = 0xA110, + UDMA_OPC_CFG_GMV_TBL = 0xA140, }; #define UDMA_QUERY_PF_CAPS_CMD_NUM 5 @@ -400,4 +402,58 @@ enum { #define UDMA_INT_NAME_LEN 32 +struct udma_cfg_gmv_tb_a { + uint32_t vf_sgid_l; + uint32_t vf_sgid_ml; + uint32_t vf_sgid_mh; + uint32_t vf_sgid_h; + uint32_t vf_type_vlan_smac; + uint32_t vf_smac_h; +}; + +struct udma_cfg_gmv_tb_b { + uint32_t vf_upi; + uint32_t vf_eid_high; + uint32_t table_idx_rsv; + uint32_t vf_id; + uint32_t resv[2]; +}; + +#define SGID_TYPE_IPV4 1 + +#define CFG_GMV_TB_VF_SGID_TYPE_S 0 +#define CFG_GMV_TB_VF_SMAC_L_S 16 +#define CFG_GMV_TB_VF_PATTERN_S 3 +#define CFG_GMV_TB_VF_SGID_TYPE_M GENMASK(1, 0) +#define CFG_GMV_TB_VF_SMAC_L_M GENMASK(31, 16) + +#define SGID_H_SHIFT 12 +#define SMAC_L_SHIFT 0 +#define SMAC_H_SHIFT 2 + +#define CFG_GMV_TBL_CMD_NUM 2 + +struct udma_eid_tbl_entry_cmd { + uint8_t resp_code; + uint8_t rsv1[3]; + uint32_t eid_addr[4]; + uint16_t eid_ad; + uint8_t rsv2[2]; +}; + +#define UDMA_EID_TB_VFID_S 0 +#define UDMA_EID_TB_VFID_M GENMASK(7, 0) +#define UDMA_EID_TB_RES_SUCCESS 0 +#define UDMA_EID_TB_RES_MODIFY 2 + +union udma_eid { + union ubcore_eid ubcore_eid; + struct { + uint32_t eid_l; + uint32_t eid_ml; + uint32_t eid_mh; + uint32_t eid_h; + } bit32_data; +}; + #endif /* _UDMA_HW_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c index 73bfd6121ed9..962a156dd812 100644 --- a/drivers/ub/hw/hns3/hns3_udma_main.c +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -21,6 +21,17 @@ #include "hns3_udma_hem.h" #include "hns3_udma_jfc.h" #include "hns3_udma_cmd.h" +static int udma_set_eid(struct ubcore_device *dev, union ubcore_eid eid) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + uint8_t port = 0; + + if (port >= udma_dev->caps.num_ports) + return -EINVAL; + + return udma_dev->hw->set_eid(udma_dev, eid); +} + static int udma_uar_alloc(struct udma_dev *udma_dev, struct udma_uar *uar) { struct udma_ida *uar_ida = &udma_dev->uar_ida; @@ -163,6 +174,7 @@ static int udma_mmap(struct ubcore_ucontext *uctx, struct vm_area_struct *vma) static struct ubcore_ops g_udma_dev_ops = { .owner = THIS_MODULE, .abi_version = 1, + .set_eid = udma_set_eid, .alloc_ucontext = udma_alloc_ucontext, .free_ucontext = udma_free_ucontext, .mmap = udma_mmap, -- Gitee From 0f04338d27d314d862ae3de51a8770cd124c73cb Mon Sep 17 00:00:00 2001 From: Weibo Zhao Date: Fri, 18 Aug 2023 16:12:15 +0800 Subject: [PATCH 14/42] hns3 udma: support of link status change driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA ----------------------------------------------------------- UDMA driver would report a event if link status changed. Driver set latest link status in event_type. Signed-off-by: Weibo Zhao --- drivers/ub/hw/hns3/hns3_udma_hw.c | 42 +++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.c b/drivers/ub/hw/hns3/hns3_udma_hw.c index f91c6f647848..72a93dfc189f 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hw.c +++ b/drivers/ub/hw/hns3/hns3_udma_hw.c @@ -16,6 +16,7 @@ #include #include #include +#include "urma/ubcore_api.h" #include "hnae3.h" #include "hns3_udma_cmd.h" #include "hns3_udma_hem.h" @@ -1674,9 +1675,50 @@ static void udma_uninit_instance(struct hnae3_handle *handle, bool reset) handle->udmainfo.instance_state = UDMA_STATE_NON_INIT; } +static void udma_link_status_change(struct hnae3_handle *handle, bool linkup) +{ + struct net_device *net_dev; + struct ubcore_event event; + struct udma_dev *dev; + uint32_t port_id; + + dev = handle->priv; + + if (IS_ERR_OR_NULL(dev)) { + pr_err("[hns3-udma:link_status_change]: Invalid dev!\n"); + return; + } + + for (port_id = 0; port_id < dev->caps.num_ports; port_id++) { + net_dev = dev->uboe.netdevs[port_id]; + if (!net_dev) { + dev_err(dev->dev, "Find netdev %u failed!\n", port_id); + return; + } + + if (net_dev == handle->udmainfo.netdev) + break; + } + + if (port_id == dev->caps.num_ports) { + dev_err(dev->dev, "Cannot find netdev!\n"); + return; + } + + if (linkup) + event.event_type = UBCORE_EVENT_PORT_ACTIVE; + else + event.event_type = UBCORE_EVENT_PORT_ERR; + + event.ub_dev = &dev->ub_dev; + event.element.port_id = port_id; + ubcore_dispatch_async_event(&event); +} + static const struct hnae3_client_ops udma_ops = { .init_instance = udma_init_instance, .uninit_instance = udma_uninit_instance, + .link_status_change = udma_link_status_change, }; static struct hnae3_client udma_client = { -- Gitee From fc19a9b64977ad09630ac219ce69295c2f338d51 Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Mon, 21 Aug 2023 16:14:49 +0800 Subject: [PATCH 15/42] hns3 udma: support create and destroy jfr. driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA ---------------------------------------------------------------- JFR(jetty for receive ) is a jetty which used to post work request(WR) and receive messages delivered by the WR of sender. Signed-off-by: Yu Zhang --- drivers/ub/hw/hns3/Makefile | 2 +- drivers/ub/hw/hns3/hns3_udma_abi.h | 16 + drivers/ub/hw/hns3/hns3_udma_common.h | 1 + drivers/ub/hw/hns3/hns3_udma_device.h | 18 ++ drivers/ub/hw/hns3/hns3_udma_jfr.c | 443 ++++++++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_jfr.h | 91 ++++++ drivers/ub/hw/hns3/hns3_udma_main.c | 3 + drivers/ub/hw/hns3/hns3_udma_qp.c | 51 +++ drivers/ub/hw/hns3/hns3_udma_qp.h | 14 + 9 files changed, 638 insertions(+), 1 deletion(-) create mode 100644 drivers/ub/hw/hns3/hns3_udma_jfr.c create mode 100644 drivers/ub/hw/hns3/hns3_udma_jfr.h diff --git a/drivers/ub/hw/hns3/Makefile b/drivers/ub/hw/hns3/Makefile index 804ab976032a..5cdbe697ee96 100644 --- a/drivers/ub/hw/hns3/Makefile +++ b/drivers/ub/hw/hns3/Makefile @@ -9,6 +9,6 @@ ccflags-y += -I$(srctree)/drivers/net/ethernet/hisilicon/hns3/ \ $(MODULE_NAME)-objs := hns3_udma_hw.o hns3_udma_main.o hns3_udma_cmd.o \ hns3_udma_hem.o hns3_udma_qp.o hns3_udma_eq.o \ - hns3_udma_db.o hns3_udma_jfc.o + hns3_udma_db.o hns3_udma_jfc.o hns3_udma_jfr.o obj-$(CONFIG_UB_UDMA_HNS3) := hns3_udma.o diff --git a/drivers/ub/hw/hns3/hns3_udma_abi.h b/drivers/ub/hw/hns3/hns3_udma_abi.h index 77c1461b9d8d..849310828190 100644 --- a/drivers/ub/hw/hns3/hns3_udma_abi.h +++ b/drivers/ub/hw/hns3/hns3_udma_abi.h @@ -19,6 +19,8 @@ #include #define MAP_COMMAND_MASK 0xff +#define UDMA_JETTY_X_PREFIX_BIT_NUM 2 +#define UDMA_JFR_QPN_PREFIX 0x1 #define UDMA_ADDR_4K_MASK 0xfffUL enum { @@ -40,6 +42,20 @@ enum udma_jfc_notify_mode { UDMA_JFC_NOTIFY_MODE_DDR_4B_ALIGN, }; +struct udma_create_jfr_ucmd { + uint64_t buf_addr; + uint64_t idx_addr; + uint64_t db_addr; +}; + +enum udma_jfr_cap_flags { + UDMA_JFR_CAP_RECORD_DB = 1 << 0, +}; + +struct udma_create_jfr_resp { + uint32_t jfr_caps; +}; + struct udma_jfc_attr_ex { uint64_t jfc_ex_mask; /* Use enum udma_jfc_init_attr_mask */ uint64_t create_flags; /* Use enum udma_jfc_create_flags */ diff --git a/drivers/ub/hw/hns3/hns3_udma_common.h b/drivers/ub/hw/hns3/hns3_udma_common.h index f9448114eca6..588eb1fa091b 100644 --- a/drivers/ub/hw/hns3/hns3_udma_common.h +++ b/drivers/ub/hw/hns3/hns3_udma_common.h @@ -21,6 +21,7 @@ #define ub_read(dev, reg) readl((dev)->reg_base + (reg)) #define UDMA_UDP_DPORT 4791 +#define DMA_WQE_SHIFT 3 #define DMA_DB_RECORD_SHIFT 1 #define udma_get_field(origin, mask, shift) \ diff --git a/drivers/ub/hw/hns3/hns3_udma_device.h b/drivers/ub/hw/hns3/hns3_udma_device.h index cab9075caf22..36d34edca569 100644 --- a/drivers/ub/hw/hns3/hns3_udma_device.h +++ b/drivers/ub/hw/hns3/hns3_udma_device.h @@ -45,6 +45,8 @@ #define UDMA_GMV_ENTRY_SZ 32 #define UDMA_CQ_BANK_NUM 4 +#define UDMA_SGE_SIZE 16 +#define UDMA_IDX_QUE_ENTRY_SZ 4 /* The minimum page size is 4K for hardware */ #define UDMA_HW_PAGE_SHIFT 12 #define UDMA_PAGE_SIZE (1 << UDMA_HW_PAGE_SHIFT) @@ -186,6 +188,12 @@ enum { TYPE_CSQ = 1 }; +enum udma_qp_state { + QPS_RESET, + QPS_RTR = 2, + QPS_RTS, + QPS_ERR = 6, +}; enum { UDMA_BUF_DIRECT = BIT(0), @@ -747,11 +755,21 @@ static inline struct udma_dev *to_udma_dev(const struct ubcore_device *ubcore_de return container_of(ubcore_dev, struct udma_dev, ub_dev); } +static inline uint32_t to_udma_hem_entries_size(uint32_t count, + uint32_t buf_shift) +{ + return udma_hw_page_align(count << buf_shift); +} static inline uint32_t to_udma_hw_page_shift(uint32_t page_shift) { return page_shift - UDMA_HW_PAGE_SHIFT; } +static inline uint64_t to_udma_hw_page_addr(uint64_t addr) +{ + return addr >> UDMA_HW_PAGE_SHIFT; +} + static inline dma_addr_t udma_buf_dma_addr(struct udma_buf *buf, uint32_t offset) { diff --git a/drivers/ub/hw/hns3/hns3_udma_jfr.c b/drivers/ub/hw/hns3/hns3_udma_jfr.c new file mode 100644 index 000000000000..d322ebfa55f8 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_jfr.c @@ -0,0 +1,443 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include +#include "hns3_udma_hem.h" +#include "hns3_udma_cmd.h" +#include "hns3_udma_db.h" +#include "hns3_udma_jfc.h" +#include "hns3_udma_jfr.h" + +static int init_jfr_cfg(struct udma_dev *dev, struct udma_jfr *jfr, + const struct ubcore_jfr_cfg *cfg) +{ + if (!cfg->max_sge || + cfg->depth > dev->caps.max_srq_wrs || + cfg->max_sge > dev->caps.max_srq_sges) { + dev_err(dev->dev, "invalid jfr cfg, depth = %u, sge = %u.\n", + cfg->depth, cfg->max_sge); + return -EINVAL; + } + + jfr->wqe_cnt = roundup_pow_of_two(cfg->depth); + + if (cfg->trans_mode == UBCORE_TP_UM) + /* reserved for UM header */ + jfr->max_sge = roundup_pow_of_two(cfg->max_sge + 1); + else + jfr->max_sge = roundup_pow_of_two(cfg->max_sge); + + memcpy(&jfr->ubcore_jfr.jfr_cfg, cfg, sizeof(struct ubcore_jfr_cfg)); + return 0; +} + +static int alloc_jfr_idx(struct udma_dev *dev, struct udma_jfr *jfr, + struct ubcore_udata *udata, uint64_t addr) +{ + struct udma_jfr_idx_que *idx_que = &jfr->idx_que; + struct udma_buf_attr buf_attr = {}; + int ret; + + jfr->idx_que.entry_shift = ilog2(UDMA_IDX_QUE_ENTRY_SZ); + + buf_attr.page_shift = dev->caps.idx_buf_pg_sz + PAGE_SHIFT; + buf_attr.region[0].size = + to_udma_hem_entries_size(jfr->wqe_cnt, + jfr->idx_que.entry_shift); + buf_attr.region[0].hopnum = dev->caps.idx_hop_num; + buf_attr.region_count = 1; + + ret = udma_mtr_create(dev, &idx_que->mtr, &buf_attr, + dev->caps.idx_ba_pg_sz + PAGE_SHIFT, + addr, !!udata); + if (ret) { + dev_err(dev->dev, + "failed to alloc JFR idx mtr, ret = %d.\n", ret); + return ret; + } + + idx_que->head = 0; + idx_que->tail = 0; + + return 0; +} + +static void free_jfr_idx(struct udma_dev *dev, struct udma_jfr *jfr) +{ + struct udma_jfr_idx_que *idx_que = &jfr->idx_que; + + return udma_mtr_destroy(dev, &idx_que->mtr); +} + +static int alloc_jfr_wqe_buf(struct udma_dev *dev, + struct udma_jfr *jfr, + struct ubcore_udata *udata, uint64_t addr) +{ + struct udma_buf_attr buf_attr = {}; + int ret; + + jfr->wqe_shift = ilog2(roundup_pow_of_two(UDMA_SGE_SIZE * + jfr->max_sge)); + + buf_attr.page_shift = dev->caps.srqwqe_buf_pg_sz + PAGE_SHIFT; + buf_attr.region[0].size = to_udma_hem_entries_size(jfr->wqe_cnt, + jfr->wqe_shift); + buf_attr.region[0].hopnum = dev->caps.srqwqe_hop_num; + buf_attr.region_count = 1; + + ret = udma_mtr_create(dev, &jfr->buf_mtr, &buf_attr, + dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT, + addr, !!udata); + if (ret) + dev_err(dev->dev, + "failed to alloc JFR buf mtr, ret = %d.\n", ret); + + return ret; +} + +static void free_jfr_wqe_buf(struct udma_dev *dev, struct udma_jfr *jfr) +{ + udma_mtr_destroy(dev, &jfr->buf_mtr); +} + +static int alloc_jfr_buf(struct udma_dev *dev, struct udma_jfr *jfr, + struct ubcore_udata *udata) +{ + struct udma_create_jfr_ucmd ucmd = {}; + struct udma_create_jfr_resp resp = {}; + int ret; + + if (udata) { + ret = copy_from_user(&ucmd, (void *)udata->udrv_data->in_addr, + min(udata->udrv_data->in_len, + (uint32_t)sizeof(ucmd))); + if (ret) { + dev_err(dev->dev, + "failed to copy JFR udata, ret = %d.\n", + ret); + return -EFAULT; + } + } + + ret = alloc_jfr_idx(dev, jfr, udata, ucmd.idx_addr); + if (ret) + return ret; + + ret = alloc_jfr_wqe_buf(dev, jfr, udata, ucmd.buf_addr); + if (ret) + goto err_idx; + + if (dev->caps.flags & UDMA_CAP_FLAG_SRQ_RECORD_DB) { + ret = udma_db_map_user(dev, ucmd.db_addr, &jfr->db); + if (ret) { + dev_err(dev->dev, + "map jfr db failed, ret = %d.\n", ret); + goto err_db; + } + jfr->jfr_caps |= UDMA_JFR_CAP_RECORD_DB; + } + + if (udata) { + resp.jfr_caps = jfr->jfr_caps; + ret = copy_to_user((void *)udata->udrv_data->out_addr, &resp, + min(udata->udrv_data->out_len, + (uint32_t)sizeof(resp))); + if (ret) { + dev_err(dev->dev, + "failed to copy jfr resp, ret = %d.\n", + ret); + goto err_copy; + } + } + refcount_set(&jfr->refcount, 1); + init_completion(&jfr->free); + return 0; + +err_copy: + if (dev->caps.flags & UDMA_CAP_FLAG_SRQ_RECORD_DB) { + udma_db_unmap_user(dev, &jfr->db); + jfr->jfr_caps &= ~UDMA_JFR_CAP_RECORD_DB; + } +err_db: + free_jfr_wqe_buf(dev, jfr); +err_idx: + free_jfr_idx(dev, jfr); + + return ret; +} + +static int udma_write_jfr_index_queue(struct udma_dev *dev, + struct udma_jfr *jfr, + struct udma_jfr_context *ctx) +{ +#define DMA_IDX_SHIFT 3 + struct udma_jfr_idx_que *idx_que = &jfr->idx_que; + uint64_t mtts_idx[MTT_MIN_COUNT] = {}; + uint64_t dma_handle_idx = 0; + int ret; + + /* Get physical address of idx que buf */ + ret = udma_mtr_find(dev, &idx_que->mtr, 0, mtts_idx, + ARRAY_SIZE(mtts_idx), &dma_handle_idx); + if (ret < 1) { + dev_err(dev->dev, "failed to find mtr for JFR idx, ret = %d.\n", + ret); + return -ENOBUFS; + } + + udma_reg_write(ctx, SRQC_IDX_HOP_NUM, + to_udma_hem_hopnum(dev->caps.idx_hop_num, jfr->wqe_cnt)); + + udma_reg_write(ctx, SRQC_IDX_BT_BA_L, dma_handle_idx >> DMA_IDX_SHIFT); + udma_reg_write(ctx, SRQC_IDX_BT_BA_H, + upper_32_bits(dma_handle_idx >> DMA_IDX_SHIFT)); + + udma_reg_write(ctx, SRQC_IDX_BA_PG_SZ, + to_udma_hw_page_shift(idx_que->mtr.hem_cfg.ba_pg_shift)); + udma_reg_write(ctx, SRQC_IDX_BUF_PG_SZ, + to_udma_hw_page_shift(idx_que->mtr.hem_cfg.buf_pg_shift)); + + udma_reg_write(ctx, SRQC_IDX_CUR_BLK_ADDR_L, + to_udma_hw_page_addr(mtts_idx[0])); + udma_reg_write(ctx, SRQC_IDX_CUR_BLK_ADDR_H, + upper_32_bits(to_udma_hw_page_addr(mtts_idx[0]))); + + udma_reg_write(ctx, SRQC_IDX_NXT_BLK_ADDR_L, + to_udma_hw_page_addr(mtts_idx[1])); + udma_reg_write(ctx, SRQC_IDX_NXT_BLK_ADDR_H, + upper_32_bits(to_udma_hw_page_addr(mtts_idx[1]))); + + return 0; +} + +static int write_jfrc(struct udma_dev *dev, struct udma_jfr *jfr, void *mb_buf) +{ + struct udma_jfr_context *ctx = (struct udma_jfr_context *)mb_buf; + uint64_t mtts_wqe[MTT_MIN_COUNT] = {}; + uint64_t dma_handle_wqe = 0; + int ret; + + memset(ctx, 0, sizeof(*ctx)); + + ret = udma_mtr_find(dev, &jfr->buf_mtr, 0, mtts_wqe, + ARRAY_SIZE(mtts_wqe), &dma_handle_wqe); + if (ret < 1) { + dev_err(dev->dev, "failed to find mtr for JFR WQE, ret = %d.\n", + ret); + return -ENOBUFS; + } + + udma_reg_write(ctx, SRQC_SRQ_ST, 1); + udma_reg_write(ctx, SRQC_SRQ_TYPE, 0); + udma_reg_write(ctx, SRQC_PD, 0); + udma_reg_write(ctx, SRQC_SRQN, jfr->jfrn); + udma_reg_write(ctx, SRQC_XRCD, 0); + udma_reg_write(ctx, SRQC_XRC_RSV, 0); + udma_reg_write(ctx, SRQC_SHIFT, ilog2(jfr->wqe_cnt)); + udma_reg_write(ctx, SRQC_RQWS, + jfr->max_sge <= 0 ? 0 : fls(jfr->max_sge - 1)); + + udma_reg_write(ctx, SRQC_WQE_HOP_NUM, + to_udma_hem_hopnum(dev->caps.srqwqe_hop_num, + jfr->wqe_cnt)); + + udma_reg_write(ctx, SRQC_WQE_BT_BA_L, dma_handle_wqe >> DMA_WQE_SHIFT); + udma_reg_write(ctx, SRQC_WQE_BT_BA_H, + upper_32_bits(dma_handle_wqe >> DMA_WQE_SHIFT)); + + udma_reg_write(ctx, SRQC_WQE_BA_PG_SZ, + to_udma_hw_page_shift(jfr->buf_mtr.hem_cfg.ba_pg_shift)); + udma_reg_write(ctx, SRQC_WQE_BUF_PG_SZ, + to_udma_hw_page_shift(jfr->buf_mtr.hem_cfg.buf_pg_shift)); + if (jfr->jfr_caps & UDMA_JFR_CAP_RECORD_DB) { + udma_reg_enable(ctx, SRQC_RECORD_DB_EN); + udma_reg_write(ctx, SRQC_RECORD_DB_ADDR_L, + lower_32_bits(jfr->db.dma) >> + DMA_DB_RECORD_SHIFT); + udma_reg_write(ctx, SRQC_RECORD_DB_ADDR_H, + upper_32_bits(jfr->db.dma)); + } + + return udma_write_jfr_index_queue(dev, jfr, ctx); +} + +static int udma_hw_create_srq(struct udma_dev *dev, + struct udma_cmd_mailbox *mailbox, + uint64_t jfrn) +{ + struct udma_cmq_desc desc; + struct udma_mbox *mb; + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mbox_desc_init(mb, mailbox->dma, 0, jfrn, UDMA_CMD_CREATE_SRQ); + + return udma_cmd_mbox(dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); +} + +static int alloc_jfrc(struct udma_dev *dev, struct udma_jfr *jfr) +{ + struct udma_jfr_table *jfr_table = &dev->jfr_table; + struct udma_ida *jfr_ida = &jfr_table->jfr_ida; + struct udma_cmd_mailbox *mailbox; + int ret; + int id; + + id = ida_alloc_range(&jfr_ida->ida, jfr_ida->min, jfr_ida->max, + GFP_KERNEL); + if (id < 0) { + dev_err(dev->dev, "failed to alloc jfr_id(%d).\n", id); + return id; + } + jfr->jfrn = (uint32_t)id; + jfr->ubcore_jfr.id = (uint32_t)id; + + ret = udma_table_get(dev, &jfr_table->table, jfr->jfrn); + if (ret) { + dev_err(dev->dev, "failed to get JFRC table, ret = %d.\n", ret); + goto err_ida; + } + + ret = xa_err(xa_store(&jfr_table->xa, jfr->jfrn, jfr, GFP_KERNEL)); + if (ret) { + dev_err(dev->dev, "failed to store JFRC, ret = %d.\n", ret); + goto err_put; + } + + mailbox = udma_alloc_cmd_mailbox(dev); + if (IS_ERR_OR_NULL(mailbox)) { + dev_err(dev->dev, "failed to alloc mailbox for JFRC.\n"); + ret = -ENOMEM; + goto err_xa; + } + + ret = write_jfrc(dev, jfr, mailbox->buf); + if (ret) { + dev_err(dev->dev, "failed to write JFRC.\n"); + goto err_mbox; + } + + ret = udma_hw_create_srq(dev, mailbox, jfr->jfrn); + if (ret) { + dev_err(dev->dev, "failed to config JFRC, ret = %d.\n", ret); + goto err_mbox; + } + + udma_free_cmd_mailbox(dev, mailbox); + + return 0; + +err_mbox: + udma_free_cmd_mailbox(dev, mailbox); +err_xa: + xa_erase(&jfr_table->xa, jfr->jfrn); +err_put: + udma_table_put(dev, &jfr_table->table, jfr->jfrn); +err_ida: + ida_free(&jfr_ida->ida, id); + + return ret; +} + +static int udma_hw_destroy_srq(struct udma_dev *dev, uint64_t jfrn) +{ + struct udma_cmq_desc desc; + struct udma_mbox *mb; + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mbox_desc_init(mb, 0, 0, jfrn, UDMA_CMD_DESTROY_SRQ); + + return udma_cmd_mbox(dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); +} + +static void free_jfrc(struct udma_dev *dev, uint32_t jfrn) +{ + struct udma_jfr_table *jfr_table = &dev->jfr_table; + int ret; + + ret = udma_hw_destroy_srq(dev, jfrn); + if (ret) + dev_err(dev->dev, "destroy failed (%d) for JFRN 0x%06x\n", + ret, jfrn); + + xa_erase(&jfr_table->xa, jfrn); + + udma_table_put(dev, &jfr_table->table, jfrn); + + ida_free(&jfr_table->jfr_ida.ida, (int)jfrn); +} + +static void free_jfr_buf(struct udma_dev *dev, struct udma_jfr *jfr) +{ + if (refcount_dec_and_test(&jfr->refcount)) + complete(&jfr->free); + wait_for_completion(&jfr->free); + + if (dev->caps.flags & UDMA_CAP_FLAG_SRQ_RECORD_DB) + udma_db_unmap_user(dev, &jfr->db); + free_jfr_wqe_buf(dev, jfr); + free_jfr_idx(dev, jfr); +} + +struct ubcore_jfr *udma_create_jfr(struct ubcore_device *dev, const struct ubcore_jfr_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + struct udma_jfr *jfr; + int ret; + + jfr = kcalloc(1, sizeof(*jfr), GFP_KERNEL); + if (!jfr) + return NULL; + + ret = init_jfr_cfg(udma_dev, jfr, cfg); + if (ret) + goto err_alloc_jfr; + + ret = alloc_jfr_buf(udma_dev, jfr, udata); + if (ret) + goto err_alloc_jfr; + + ret = alloc_jfrc(udma_dev, jfr); + if (ret) + goto err_alloc_buf; + + xa_init(&jfr->tp_table_xa); + init_jetty_x_qpn_bitmap(udma_dev, &jfr->qpn_map, + udma_dev->caps.num_jfr_shift, + UDMA_JFR_QPN_PREFIX, jfr->jfrn); + return &jfr->ubcore_jfr; + +err_alloc_buf: + free_jfr_buf(udma_dev, jfr); +err_alloc_jfr: + kfree(jfr); + + return NULL; +} + +int udma_destroy_jfr(struct ubcore_jfr *jfr) +{ + struct udma_dev *dev = to_udma_dev(jfr->ub_dev); + struct udma_jfr *udma_jfr = to_udma_jfr(jfr); + + clean_jetty_x_qpn_bitmap(&udma_jfr->qpn_map); + free_jfrc(dev, udma_jfr->jfrn); + free_jfr_buf(dev, udma_jfr); + kfree(jfr); + + return 0; +} diff --git a/drivers/ub/hw/hns3/hns3_udma_jfr.h b/drivers/ub/hw/hns3/hns3_udma_jfr.h new file mode 100644 index 000000000000..59b7ea06ec0f --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_jfr.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_JFR_H +#define _UDMA_JFR_H + +#include "hns3_udma_common.h" +#include "hns3_udma_qp.h" + +struct udma_jfr_idx_que { + struct udma_mtr mtr; + int entry_shift; + uint32_t head; + uint32_t tail; +}; + +struct udma_jfr { + struct ubcore_jfr ubcore_jfr; + uint32_t jfrn; + uint32_t wqe_cnt; + uint32_t max_sge; + uint32_t wqe_shift; + struct udma_jfr_idx_que idx_que; + struct udma_mtr buf_mtr; + struct udma_db db; + struct udma_qp *um_qp; + struct xarray tp_table_xa; + + refcount_t refcount; + struct completion free; + + struct udma_qpn_bitmap qpn_map; + void (*event)(struct udma_jfr *jfr, enum udma_event event_type); + uint32_t jfr_caps; +}; + +struct udma_jfr_context { + uint32_t data[16]; +}; + +#define SRQC_FIELD_LOC(h, l) ((uint64_t)(h) << 32 | (l)) + +#define SRQC_SRQ_ST SRQC_FIELD_LOC(1, 0) +#define SRQC_WQE_HOP_NUM SRQC_FIELD_LOC(3, 2) +#define SRQC_SHIFT SRQC_FIELD_LOC(7, 4) +#define SRQC_SRQN SRQC_FIELD_LOC(31, 8) +#define SRQC_LIMIT_WL SRQC_FIELD_LOC(47, 32) +#define SRQC_XRCD SRQC_FIELD_LOC(87, 64) +#define SRQC_WQE_BT_BA_L SRQC_FIELD_LOC(159, 128) +#define SRQC_WQE_BT_BA_H SRQC_FIELD_LOC(188, 160) +#define SRQC_SRQ_TYPE SRQC_FIELD_LOC(191, 191) +#define SRQC_PD SRQC_FIELD_LOC(215, 192) +#define SRQC_RQWS SRQC_FIELD_LOC(219, 216) +#define SRQC_IDX_BT_BA_L SRQC_FIELD_LOC(255, 224) +#define SRQC_IDX_BT_BA_H SRQC_FIELD_LOC(284, 256) +#define SRQC_IDX_CUR_BLK_ADDR_L SRQC_FIELD_LOC(319, 288) +#define SRQC_IDX_CUR_BLK_ADDR_H SRQC_FIELD_LOC(339, 320) +#define SRQC_IDX_HOP_NUM SRQC_FIELD_LOC(343, 342) +#define SRQC_IDX_BA_PG_SZ SRQC_FIELD_LOC(347, 344) +#define SRQC_IDX_BUF_PG_SZ SRQC_FIELD_LOC(351, 348) +#define SRQC_IDX_NXT_BLK_ADDR_L SRQC_FIELD_LOC(383, 352) +#define SRQC_IDX_NXT_BLK_ADDR_H SRQC_FIELD_LOC(403, 384) +#define SRQC_XRC_RSV SRQC_FIELD_LOC(439, 416) +#define SRQC_WQE_BA_PG_SZ SRQC_FIELD_LOC(443, 440) +#define SRQC_WQE_BUF_PG_SZ SRQC_FIELD_LOC(447, 444) +#define SRQC_RECORD_DB_EN SRQC_FIELD_LOC(448, 448) +#define SRQC_RECORD_DB_ADDR_L SRQC_FIELD_LOC(479, 449) +#define SRQC_RECORD_DB_ADDR_H SRQC_FIELD_LOC(511, 480) + +static inline struct udma_jfr *to_udma_jfr(struct ubcore_jfr *ubcore_jfr) +{ + return container_of(ubcore_jfr, struct udma_jfr, ubcore_jfr); +} + +struct ubcore_jfr *udma_create_jfr(struct ubcore_device *dev, const struct ubcore_jfr_cfg *cfg, + struct ubcore_udata *udata); +int udma_destroy_jfr(struct ubcore_jfr *jfr); + +#endif /* _UDMA_JFR_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c index 962a156dd812..d78a9cef1e48 100644 --- a/drivers/ub/hw/hns3/hns3_udma_main.c +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -19,6 +19,7 @@ #include "hns3_udma_abi.h" #include "hns3_udma_device.h" #include "hns3_udma_hem.h" +#include "hns3_udma_jfr.h" #include "hns3_udma_jfc.h" #include "hns3_udma_cmd.h" static int udma_set_eid(struct ubcore_device *dev, union ubcore_eid eid) @@ -181,6 +182,8 @@ static struct ubcore_ops g_udma_dev_ops = { .create_jfc = udma_create_jfc, .modify_jfc = udma_modify_jfc, .destroy_jfc = udma_destroy_jfc, + .create_jfr = udma_create_jfr, + .destroy_jfr = udma_destroy_jfr, }; static void udma_cleanup_uar_table(struct udma_dev *dev) diff --git a/drivers/ub/hw/hns3/hns3_udma_qp.c b/drivers/ub/hw/hns3/hns3_udma_qp.c index 2258149dfb1b..b8909731fbad 100644 --- a/drivers/ub/hw/hns3/hns3_udma_qp.c +++ b/drivers/ub/hw/hns3/hns3_udma_qp.c @@ -14,9 +14,60 @@ */ #include +#include "hns3_udma_abi.h" #include "hns3_udma_jfc.h" #include "hns3_udma_hem.h" #include "hns3_udma_qp.h" +static void init_qpn_bitmap(struct udma_qpn_bitmap *qpn_map, uint32_t qpn_shift) +{ + int i; + + qpn_map->qpn_shift = qpn_shift; + mutex_init(&qpn_map->bank_mutex); + /* reserved 0 for UD */ + qpn_map->bank[0].min = 1; + qpn_map->bank[0].inuse = 1; + qpn_map->bank[0].next = qpn_map->bank[0].min; + for (i = 0; i < UDMA_QP_BANK_NUM; i++) { + ida_init(&qpn_map->bank[i].ida); + qpn_map->bank[i].max = (1 << qpn_shift) / UDMA_QP_BANK_NUM - 1; + } +} + +void init_jetty_x_qpn_bitmap(struct udma_dev *dev, + struct udma_qpn_bitmap *qpn_map, + uint32_t jetty_x_shift, + uint32_t prefix, uint32_t jid) +{ +#define QPN_SHIFT_MIN 3 + int qpn_shift; + + qpn_shift = dev->caps.num_qps_shift - jetty_x_shift - + UDMA_JETTY_X_PREFIX_BIT_NUM; + if (qpn_shift <= QPN_SHIFT_MIN) { + qpn_map->qpn_shift = 0; + return; + } + + qpn_map->qpn_prefix = prefix << + (dev->caps.num_qps_shift - + UDMA_JETTY_X_PREFIX_BIT_NUM); + qpn_map->jid = jid; + init_qpn_bitmap(qpn_map, qpn_shift); +} + +void clean_jetty_x_qpn_bitmap(struct udma_qpn_bitmap *qpn_map) +{ + int i; + + if (!qpn_map->qpn_shift) + return; + mutex_lock(&qpn_map->bank_mutex); + for (i = 0; i < UDMA_QP_BANK_NUM; i++) + ida_destroy(&qpn_map->bank[i].ida); + mutex_unlock(&qpn_map->bank_mutex); +} + static inline uint8_t get_qp_bankid(uint64_t qpn) { /* The lower 3 bits of QPN are used to hash to different banks */ diff --git a/drivers/ub/hw/hns3/hns3_udma_qp.h b/drivers/ub/hw/hns3/hns3_udma_qp.h index bd0be4cb27f1..479163b7757d 100644 --- a/drivers/ub/hw/hns3/hns3_udma_qp.h +++ b/drivers/ub/hw/hns3/hns3_udma_qp.h @@ -18,6 +18,15 @@ #include "hns3_udma_device.h" +struct udma_qpn_bitmap { + uint32_t qpn_prefix; + uint32_t jid; + uint32_t qpn_shift; + struct udma_bank bank[UDMA_QP_BANK_NUM]; + struct mutex bank_mutex; + atomic_t ref_num; +}; + struct udma_qp { struct udma_dev *udma_device; struct udma_mtr mtr; @@ -29,6 +38,11 @@ struct udma_qp { struct completion free; }; +void init_jetty_x_qpn_bitmap(struct udma_dev *dev, + struct udma_qpn_bitmap *qpn_map, + uint32_t jetty_x_shift, uint32_t prefix, + uint32_t jid); +void clean_jetty_x_qpn_bitmap(struct udma_qpn_bitmap *qpn_map); void udma_qp_event(struct udma_dev *udma_dev, uint32_t qpn, int event_type); #endif /* _UDMA_QP_H */ -- Gitee From 8d47f8c2f40984b9a875605bd2c1b10f41bd8bb1 Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Mon, 21 Aug 2023 20:49:36 +0800 Subject: [PATCH 16/42] hns3 udma: support report events to AE or CE handler. driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA --------------------------------------------------------------------- When an AE(asynchronous event) or CE(completion event) interrupt is triggered, the UDMA Linux driver finally reports them to corresponding URMA handler. Signed-off-by: Yu Zhang --- drivers/ub/hw/hns3/hns3_udma_eq.c | 13 +++++++- drivers/ub/hw/hns3/hns3_udma_jfc.c | 51 ++++++++++++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_jfc.h | 2 ++ drivers/ub/hw/hns3/hns3_udma_jfr.c | 32 +++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_jfr.h | 1 + 5 files changed, 98 insertions(+), 1 deletion(-) diff --git a/drivers/ub/hw/hns3/hns3_udma_eq.c b/drivers/ub/hw/hns3/hns3_udma_eq.c index 686be0cc8c55..a1beecbd02a4 100644 --- a/drivers/ub/hw/hns3/hns3_udma_eq.c +++ b/drivers/ub/hw/hns3/hns3_udma_eq.c @@ -16,8 +16,10 @@ #include #include "hnae3.h" #include "hns3_udma_hem.h" -#include "hns3_udma_eq.h" +#include "hns3_udma_jfc.h" +#include "hns3_udma_jfr.h" #include "hns3_udma_qp.h" +#include "hns3_udma_eq.h" static int alloc_eq_buf(struct udma_dev *udma_dev, struct udma_eq *eq) { @@ -209,6 +211,13 @@ static void aeq_event_report(struct udma_dev *udma_dev, case UDMA_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: udma_qp_event(udma_dev, queue_num, event_type); break; + case UDMA_EVENT_TYPE_JFR_LIMIT_REACH: + udma_jfr_event(udma_dev, queue_num, event_type); + break; + case UDMA_EVENT_TYPE_JFC_ACCESS_ERROR: + case UDMA_EVENT_TYPE_JFC_OVERFLOW: + udma_jfc_event(udma_dev, queue_num, event_type); + break; case UDMA_EVENT_TYPE_MB: udma_cmd_event(udma_dev, le16_to_cpu(aeqe->event.cmd.token), @@ -369,6 +378,8 @@ static int udma_ceq_int(struct udma_dev *udma_dev, cqn = udma_get_field(ceqe->comp, UDMA_CEQE_COMP_CQN_M, UDMA_CEQE_COMP_CQN_S); + udma_jfc_completion(udma_dev, cqn); + ++eq->cons_index; ceqe_found = 1; diff --git a/drivers/ub/hw/hns3/hns3_udma_jfc.c b/drivers/ub/hw/hns3/hns3_udma_jfc.c index e1b3aa3009d6..5be01495c036 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jfc.c +++ b/drivers/ub/hw/hns3/hns3_udma_jfc.c @@ -592,3 +592,54 @@ int udma_modify_jfc(struct ubcore_jfc *ubcore_jfc, const struct ubcore_jfc_attr return ret; } + +void udma_jfc_completion(struct udma_dev *udma_dev, uint32_t cqn) +{ + struct ubcore_jfc *ubcore_jfc; + struct udma_jfc *udma_jfc; + + udma_jfc = (struct udma_jfc *)xa_load(&udma_dev->jfc_table.xa, cqn); + if (!udma_jfc) { + dev_warn(udma_dev->dev, + "Completion event for bogus CQ 0x%06x\n", cqn); + return; + } + + ubcore_jfc = &udma_jfc->ubcore_jfc; + if (ubcore_jfc->jfce_handler) + ubcore_jfc->jfce_handler(ubcore_jfc); +} + +void udma_jfc_event(struct udma_dev *udma_dev, uint32_t cqn, int event_type) +{ + struct device *dev = udma_dev->dev; + struct udma_jfc *udma_jfc; + struct ubcore_jfc *ubcore_jfc; + struct ubcore_event event; + + udma_jfc = (struct udma_jfc *)xa_load(&udma_dev->jfc_table.xa, cqn); + if (!udma_jfc) { + dev_warn(dev, "Async event for bogus CQ 0x%06x\n", cqn); + return; + } + + if (event_type != UDMA_EVENT_TYPE_JFC_ACCESS_ERROR && + event_type != UDMA_EVENT_TYPE_JFC_OVERFLOW) { + dev_err(dev, "Unexpected event type 0x%x on CQ 0x%06x\n", + event_type, cqn); + return; + } + + refcount_inc(&udma_jfc->refcount); + + ubcore_jfc = &udma_jfc->ubcore_jfc; + if (ubcore_jfc->jfae_handler) { + event.ub_dev = ubcore_jfc->ub_dev; + event.element.jfc = ubcore_jfc; + event.event_type = UBCORE_EVENT_JFC_ERR; + ubcore_jfc->jfae_handler(&event, ubcore_jfc->uctx); + } + + if (refcount_dec_and_test(&udma_jfc->refcount)) + complete(&udma_jfc->free); +} diff --git a/drivers/ub/hw/hns3/hns3_udma_jfc.h b/drivers/ub/hw/hns3/hns3_udma_jfc.h index ddfd56575141..5af78b398aea 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jfc.h +++ b/drivers/ub/hw/hns3/hns3_udma_jfc.h @@ -72,6 +72,8 @@ struct ubcore_jfc *udma_create_jfc(struct ubcore_device *dev, const struct ubcor int udma_destroy_jfc(struct ubcore_jfc *jfc); int udma_modify_jfc(struct ubcore_jfc *ubcore_jfc, const struct ubcore_jfc_attr *attr, struct ubcore_udata *udata); +void udma_jfc_completion(struct udma_dev *udma_dev, uint32_t cqn); +void udma_jfc_event(struct udma_dev *udma_dev, uint32_t cqn, int event_type); static inline uint8_t get_jfc_bankid(uint64_t cqn) { /* The lower 2 bits of CQN are used to hash to different banks */ diff --git a/drivers/ub/hw/hns3/hns3_udma_jfr.c b/drivers/ub/hw/hns3/hns3_udma_jfr.c index d322ebfa55f8..bdaee22485a3 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jfr.c +++ b/drivers/ub/hw/hns3/hns3_udma_jfr.c @@ -441,3 +441,35 @@ int udma_destroy_jfr(struct ubcore_jfr *jfr) return 0; } + +void udma_jfr_event(struct udma_dev *udma_dev, uint32_t jfrn, int event_type) +{ + struct udma_jfr_table *jfr_table = &udma_dev->jfr_table; + struct ubcore_jfr *ubcore_jfr; + struct udma_jfr *jfr; + struct ubcore_event event; + + xa_lock(&jfr_table->xa); + jfr = (struct udma_jfr *)xa_load(&jfr_table->xa, jfrn); + xa_unlock(&jfr_table->xa); + + if (!jfr) { + dev_warn(udma_dev->dev, "Async event for bogus SRQ 0x%08x\n", + jfrn); + return; + } + + event.event_type = UBCORE_EVENT_JFR_ACCESS_ERR; + + refcount_inc(&jfr->refcount); + ubcore_jfr = &jfr->ubcore_jfr; + if (ubcore_jfr->jfae_handler) { + event.ub_dev = ubcore_jfr->ub_dev; + event.element.jfr = ubcore_jfr; + ubcore_jfr->jfae_handler(&event, ubcore_jfr->uctx); + dev_info(udma_dev->dev, "Async event for JFR 0x%08x\n", jfrn); + } + + if (refcount_dec_and_test(&jfr->refcount)) + complete(&jfr->free); +} diff --git a/drivers/ub/hw/hns3/hns3_udma_jfr.h b/drivers/ub/hw/hns3/hns3_udma_jfr.h index 59b7ea06ec0f..3757aac10cb7 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jfr.h +++ b/drivers/ub/hw/hns3/hns3_udma_jfr.h @@ -87,5 +87,6 @@ static inline struct udma_jfr *to_udma_jfr(struct ubcore_jfr *ubcore_jfr) struct ubcore_jfr *udma_create_jfr(struct ubcore_device *dev, const struct ubcore_jfr_cfg *cfg, struct ubcore_udata *udata); int udma_destroy_jfr(struct ubcore_jfr *jfr); +void udma_jfr_event(struct udma_dev *udma_dev, uint32_t jfrn, int event_type); #endif /* _UDMA_JFR_H */ -- Gitee From e94ccb1b43b0367e2f2cf2555318919374c23aad Mon Sep 17 00:00:00 2001 From: Weibo Zhao Date: Mon, 21 Aug 2023 16:21:54 +0800 Subject: [PATCH 17/42] hns3 udma: add register and unregister segment driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA ----------------------------------------------------- Segment is a memory that alloced by user. User calls register segment to allow hardware to read / write data in memory. Driver should create a memory translate and protect table of segment. Signed-off-by: Weibo Zhao --- drivers/ub/hw/hns3/Makefile | 3 +- drivers/ub/hw/hns3/hns3_udma_abi.h | 1 + drivers/ub/hw/hns3/hns3_udma_device.h | 18 ++ drivers/ub/hw/hns3/hns3_udma_main.c | 3 + drivers/ub/hw/hns3/hns3_udma_segment.c | 313 +++++++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_segment.h | 84 +++++++ 6 files changed, 421 insertions(+), 1 deletion(-) create mode 100644 drivers/ub/hw/hns3/hns3_udma_segment.c create mode 100644 drivers/ub/hw/hns3/hns3_udma_segment.h diff --git a/drivers/ub/hw/hns3/Makefile b/drivers/ub/hw/hns3/Makefile index 5cdbe697ee96..2ef23a914826 100644 --- a/drivers/ub/hw/hns3/Makefile +++ b/drivers/ub/hw/hns3/Makefile @@ -9,6 +9,7 @@ ccflags-y += -I$(srctree)/drivers/net/ethernet/hisilicon/hns3/ \ $(MODULE_NAME)-objs := hns3_udma_hw.o hns3_udma_main.o hns3_udma_cmd.o \ hns3_udma_hem.o hns3_udma_qp.o hns3_udma_eq.o \ - hns3_udma_db.o hns3_udma_jfc.o hns3_udma_jfr.o + hns3_udma_db.o hns3_udma_jfc.o hns3_udma_jfr.o \ + hns3_udma_segment.o obj-$(CONFIG_UB_UDMA_HNS3) := hns3_udma.o diff --git a/drivers/ub/hw/hns3/hns3_udma_abi.h b/drivers/ub/hw/hns3/hns3_udma_abi.h index 849310828190..a7d0a18ebaab 100644 --- a/drivers/ub/hw/hns3/hns3_udma_abi.h +++ b/drivers/ub/hw/hns3/hns3_udma_abi.h @@ -22,6 +22,7 @@ #define UDMA_JETTY_X_PREFIX_BIT_NUM 2 #define UDMA_JFR_QPN_PREFIX 0x1 #define UDMA_ADDR_4K_MASK 0xfffUL +#define URMA_SEG_ACCESS_GUARD (1UL << 5) enum { UDMA_MMAP_UAR_PAGE, diff --git a/drivers/ub/hw/hns3/hns3_udma_device.h b/drivers/ub/hw/hns3/hns3_udma_device.h index 36d34edca569..f12eb0c2985a 100644 --- a/drivers/ub/hw/hns3/hns3_udma_device.h +++ b/drivers/ub/hw/hns3/hns3_udma_device.h @@ -719,6 +719,19 @@ struct udma_dev { spinlock_t dip_list_lock; }; +struct udma_seg { + struct ubcore_target_seg ubcore_seg; + uint64_t iova; + uint64_t size; + uint32_t key; + uint32_t pd; + uint32_t access; + int enabled; + uint32_t pbl_hop_num; + struct udma_mtr pbl_mtr; + uint32_t npages; +}; + static inline void *udma_buf_offset(struct udma_buf *buf, uint32_t offset) { @@ -755,6 +768,11 @@ static inline struct udma_dev *to_udma_dev(const struct ubcore_device *ubcore_de return container_of(ubcore_dev, struct udma_dev, ub_dev); } +static inline struct udma_seg *to_udma_seg(struct ubcore_target_seg *seg) +{ + return container_of(seg, struct udma_seg, ubcore_seg); +} + static inline uint32_t to_udma_hem_entries_size(uint32_t count, uint32_t buf_shift) { diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c index d78a9cef1e48..77d6979e4197 100644 --- a/drivers/ub/hw/hns3/hns3_udma_main.c +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -21,6 +21,7 @@ #include "hns3_udma_hem.h" #include "hns3_udma_jfr.h" #include "hns3_udma_jfc.h" +#include "hns3_udma_segment.h" #include "hns3_udma_cmd.h" static int udma_set_eid(struct ubcore_device *dev, union ubcore_eid eid) { @@ -179,6 +180,8 @@ static struct ubcore_ops g_udma_dev_ops = { .alloc_ucontext = udma_alloc_ucontext, .free_ucontext = udma_free_ucontext, .mmap = udma_mmap, + .register_seg = udma_register_seg, + .unregister_seg = udma_unregister_seg, .create_jfc = udma_create_jfc, .modify_jfc = udma_modify_jfc, .destroy_jfc = udma_destroy_jfc, diff --git a/drivers/ub/hw/hns3/hns3_udma_segment.c b/drivers/ub/hw/hns3/hns3_udma_segment.c new file mode 100644 index 000000000000..e4a723d585a5 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_segment.c @@ -0,0 +1,313 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include "urma/ubcore_types.h" +#include "hns3_udma_abi.h" +#include "hns3_udma_hem.h" +#include "hns3_udma_cmd.h" +#include "hns3_udma_segment.h" + +static uint32_t hw_index_to_key(int ind) +{ + return ((uint32_t)ind << SEG_KEY_OFFSET); +} + +uint64_t key_to_hw_index(uint32_t key) +{ + return (key >> SEG_KEY_OFFSET); +} + +static int udma_hw_create_mpt(struct udma_dev *udma_dev, + struct udma_cmd_mailbox *mailbox, + uint64_t mpt_index) +{ + struct udma_cmq_desc desc; + struct udma_mbox *mb; + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mbox_desc_init(mb, mailbox->dma, 0, mpt_index, UDMA_CMD_CREATE_MPT); + + return udma_cmd_mbox(udma_dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); +} + +int udma_hw_destroy_mpt(struct udma_dev *udma_dev, + struct udma_cmd_mailbox *mailbox, + uint64_t mpt_index) +{ + struct udma_cmq_desc desc; + struct udma_mbox *mb; + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mbox_desc_init(mb, 0, 0, mpt_index, UDMA_CMD_DESTROY_MPT); + + return udma_cmd_mbox(udma_dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); +} + +static int alloc_seg_key(struct udma_dev *udma_dev, struct udma_seg *seg) +{ + struct udma_ida *seg_ida = &udma_dev->seg_table.seg_ida; + int err; + int id; + + id = ida_alloc_range(&seg_ida->ida, seg_ida->min, seg_ida->max, + GFP_KERNEL); + if (id < 0) { + dev_err(udma_dev->dev, "failed to alloc id for MR key, id(%d)\n", + id); + return -ENOMEM; + } + + seg->key = hw_index_to_key(id); + + err = udma_table_get(udma_dev, &udma_dev->seg_table.table, + (uint64_t)id); + if (err) { + dev_err(udma_dev->dev, + "failed to alloc mtpt, ret = %d.\n", err); + goto err_free_bitmap; + } + + return 0; +err_free_bitmap: + ida_free(&seg_ida->ida, id); + return err; +} + +static int alloc_seg_pbl(struct udma_dev *udma_dev, struct udma_seg *seg, + bool is_user) +{ + struct udma_buf_attr buf_attr = {}; + int err; + + seg->pbl_hop_num = udma_dev->caps.pbl_hop_num; + buf_attr.page_shift = udma_dev->caps.pbl_buf_pg_sz + PAGE_SHIFT; + buf_attr.region[0].size = seg->size; + buf_attr.region[0].hopnum = seg->pbl_hop_num; + buf_attr.region_count = 1; + buf_attr.mtt_only = false; + + err = udma_mtr_create(udma_dev, &seg->pbl_mtr, &buf_attr, + udma_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT, + seg->iova, is_user); + if (err) + dev_err(udma_dev->dev, "failed to alloc pbl mtr, ret = %d.\n", + err); + else + seg->npages = seg->pbl_mtr.hem_cfg.buf_pg_count; + + return err; +} + +static int set_mtpt_pbl(struct udma_dev *udma_dev, + struct udma_mpt_entry *mpt_entry, + struct udma_seg *seg) +{ + uint64_t pages[UDMA_MAX_INNER_MTPT_NUM] = {}; + uint64_t pbl_ba; + int i, count; + + count = udma_mtr_find(udma_dev, &seg->pbl_mtr, 0, pages, + min_t(int, ARRAY_SIZE(pages), seg->npages), + &pbl_ba); + if (count < 1) { + dev_err(udma_dev->dev, "failed to find PBL mtr, count = %d.\n", + count); + return -ENOBUFS; + } + + /* Aligned to the hardware address access unit */ + for (i = 0; i < count; i++) + pages[i] >>= PA_PAGE_SHIFT; + + mpt_entry->pbl_size = cpu_to_le32(seg->npages); + mpt_entry->pbl_ba_l = cpu_to_le32(pbl_ba >> MPT_PAGE_OFFSET); + udma_reg_write(mpt_entry, MPT_PBL_BA_H, + upper_32_bits(pbl_ba >> MPT_PAGE_OFFSET)); + mpt_entry->pa0_l = cpu_to_le32(lower_32_bits(pages[0])); + udma_reg_write(mpt_entry, MPT_PA0_H, upper_32_bits(pages[0])); + mpt_entry->pa1_l = cpu_to_le32(lower_32_bits(pages[1])); + udma_reg_write(mpt_entry, MPT_PA1_H, upper_32_bits(pages[1])); + udma_reg_write(mpt_entry, MPT_PBL_BUF_PG_SZ, + to_hr_hw_page_shift(seg->pbl_mtr.hem_cfg.buf_pg_shift)); + + return 0; +} + +static int udma_write_seg_mpt(struct udma_dev *udma_dev, + void *mb_buf, struct udma_seg *seg) +{ + struct udma_mpt_entry *mpt_entry; + int ret = 0; + + mpt_entry = (struct udma_mpt_entry *)mb_buf; + memset(mpt_entry, 0, sizeof(*mpt_entry)); + + udma_reg_write(mpt_entry, MPT_ST, MPT_ST_VALID); + udma_reg_write(mpt_entry, MPT_PD, seg->pd); + udma_reg_enable(mpt_entry, MPT_L_INV_EN); + + udma_reg_write(mpt_entry, MPT_RW_EN, + !!(seg->access & UBCORE_ACCESS_REMOTE_WRITE)); + udma_reg_write(mpt_entry, MPT_LW_EN, + !!(seg->access & UBCORE_ACCESS_LOCAL_WRITE)); + udma_reg_write(mpt_entry, MPT_R_INV_EN, + !!(seg->access & UBCORE_ACCESS_REMOTE_INVALIDATE)); + + mpt_entry->len_l = cpu_to_le32(lower_32_bits(seg->size)); + mpt_entry->len_h = cpu_to_le32(upper_32_bits(seg->size)); + mpt_entry->lkey = cpu_to_le32(seg->key); + mpt_entry->va_l = cpu_to_le32(lower_32_bits(seg->iova)); + mpt_entry->va_h = cpu_to_le32(upper_32_bits(seg->iova)); + + udma_reg_write(mpt_entry, MPT_PERSIST_EN, 1); + + if (seg->pbl_hop_num != UDMA_HOP_NUM_0) + udma_reg_write(mpt_entry, MPT_PBL_HOP_NUM, seg->pbl_hop_num); + + udma_reg_write(mpt_entry, MPT_PBL_BA_PG_SZ, + to_hr_hw_page_shift(seg->pbl_mtr.hem_cfg.ba_pg_shift)); + udma_reg_enable(mpt_entry, MPT_INNER_PA_VLD); + + ret = set_mtpt_pbl(udma_dev, mpt_entry, seg); + + return ret; +} + +static int udma_seg_enable(struct udma_dev *udma_dev, struct udma_seg *seg) +{ + uint64_t seg_idx = key_to_hw_index(seg->key); + struct device *dev = udma_dev->dev; + struct udma_cmd_mailbox *mailbox; + int ret; + + /* Allocate mailbox memory */ + mailbox = udma_alloc_cmd_mailbox(udma_dev); + if (IS_ERR(mailbox)) { + ret = PTR_ERR(mailbox); + return ret; + } + + ret = udma_write_seg_mpt(udma_dev, mailbox->buf, seg); + if (ret) { + dev_err(dev, "failed to write mtpt, ret = %d.\n", ret); + goto err_page; + } + + ret = udma_hw_create_mpt(udma_dev, mailbox, + seg_idx & (udma_dev->caps.num_mtpts - 1)); + if (ret) { + dev_err(dev, "failed to create mpt, ret = %d.\n", ret); + goto err_page; + } + +err_page: + udma_free_cmd_mailbox(udma_dev, mailbox); + + return ret; +} + +static void free_seg_pbl(struct udma_dev *udma_dev, struct udma_seg *seg) +{ + udma_mtr_destroy(udma_dev, &seg->pbl_mtr); +} + +static void free_seg_key(struct udma_dev *udma_dev, struct udma_seg *seg) +{ + uint64_t obj = key_to_hw_index(seg->key); + + udma_table_put(udma_dev, &udma_dev->seg_table.table, obj); + ida_free(&udma_dev->seg_table.seg_ida.ida, (int)obj); +} + +struct ubcore_target_seg *udma_register_seg(struct ubcore_device *dev, + const struct ubcore_seg_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + struct udma_ucontext *udma_ctx; + struct udma_seg *seg; + int ret; + + if (cfg->flag.bs.access >= URMA_SEG_ACCESS_GUARD) { + dev_err(udma_dev->dev, "Invalid segment access 0x%x.\n", + cfg->flag.bs.access); + return NULL; + } + + seg = kcalloc(1, sizeof(*seg), GFP_KERNEL); + if (!seg) + return NULL; + + udma_ctx = to_udma_ucontext(udata->uctx); + seg->iova = cfg->va; + seg->size = cfg->len; + seg->pd = udma_ctx->pdn; + seg->access = cfg->flag.bs.access; + + ret = alloc_seg_key(udma_dev, seg); + if (ret) + goto err_alloc_key; + + ret = alloc_seg_pbl(udma_dev, seg, !!udata); + if (ret) + goto err_alloc_pbl; + + ret = udma_seg_enable(udma_dev, seg); + if (ret) + goto err_enable_seg; + seg->enabled = 1; + seg->ubcore_seg.seg.key_id = seg->key; + + + return &seg->ubcore_seg; + +err_enable_seg: + free_seg_pbl(udma_dev, seg); +err_alloc_pbl: + free_seg_key(udma_dev, seg); +err_alloc_key: + kfree(seg); + return NULL; +} + +void udma_seg_free(struct udma_dev *udma_dev, struct udma_seg *seg) +{ + int ret; + + if (seg->enabled) { + ret = udma_hw_destroy_mpt(udma_dev, NULL, + key_to_hw_index(seg->key) & + (udma_dev->caps.num_mtpts - 1)); + if (ret) + dev_warn(udma_dev->dev, "failed to destroy mpt, ret = %d.\n", + ret); + } + + free_seg_pbl(udma_dev, seg); + free_seg_key(udma_dev, seg); +} + +int udma_unregister_seg(struct ubcore_target_seg *seg) +{ + struct udma_dev *udma_dev = to_udma_dev(seg->ub_dev); + struct udma_seg *udma_seg = to_udma_seg(seg); + + udma_seg_free(udma_dev, udma_seg); + kfree(udma_seg); + + return 0; +} diff --git a/drivers/ub/hw/hns3/hns3_udma_segment.h b/drivers/ub/hw/hns3/hns3_udma_segment.h new file mode 100644 index 000000000000..37eb67d5f795 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_segment.h @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_SEGMENT_H +#define _UDMA_SEGMENT_H + +#include +#include +#include +#include "urma/ubcore_opcode.h" +#include "hns3_udma_device.h" +#include "hns3_udma_common.h" + +#define SEG_KEY_OFFSET 8 +#define MPT_PAGE_OFFSET 3 +#define PA_PAGE_SHIFT 6 +#define MPT_VA_H_SHIFT 32 +#define MPT_LEN_H_SHIFT 32 + +enum { + MPT_ST_VALID = 0x1, +}; + +struct udma_mpt_entry { + uint32_t mpt_context1[4]; + uint32_t len_l; + uint32_t len_h; + uint32_t lkey; + uint32_t va_l; + uint32_t va_h; + uint32_t pbl_size; + uint32_t pbl_ba_l; + uint32_t mpt_context2; + uint32_t pa0_l; + uint32_t mpt_context3; + uint32_t pa1_l; + uint32_t mpt_context4; +}; + +#define MPT_FIELD_LOC(h, l) ((uint64_t)(h) << 32 | (l)) + +#define MPT_ST MPT_FIELD_LOC(1, 0) +#define MPT_PBL_HOP_NUM MPT_FIELD_LOC(3, 2) +#define MPT_PBL_BA_PG_SZ MPT_FIELD_LOC(7, 4) +#define MPT_PD MPT_FIELD_LOC(31, 8) +#define MPT_R_INV_EN MPT_FIELD_LOC(33, 33) +#define MPT_L_INV_EN MPT_FIELD_LOC(34, 34) +#define MPT_RW_EN MPT_FIELD_LOC(38, 38) +#define MPT_LW_EN MPT_FIELD_LOC(39, 39) +#define MPT_PA MPT_FIELD_LOC(65, 65) +#define MPT_INNER_PA_VLD MPT_FIELD_LOC(71, 71) +#define MPT_LEN_L MPT_FIELD_LOC(159, 128) +#define MPT_LEN_H MPT_FIELD_LOC(191, 160) +#define MPT_LKEY MPT_FIELD_LOC(223, 192) +#define MPT_VA_L MPT_FIELD_LOC(255, 224) +#define MPT_VA_H MPT_FIELD_LOC(287, 256) +#define MPT_PBL_BA_H MPT_FIELD_LOC(380, 352) +#define MPT_PA0_H MPT_FIELD_LOC(441, 416) +#define MPT_PA1_L MPT_FIELD_LOC(579, 448) +#define MPT_PA1_H MPT_FIELD_LOC(505, 480) +#define MPT_PERSIST_EN MPT_FIELD_LOC(506, 506) +#define MPT_PBL_BUF_PG_SZ MPT_FIELD_LOC(511, 508) + +#define UDMA_MAX_INNER_MTPT_NUM 2 + +struct ubcore_target_seg *udma_register_seg(struct ubcore_device *dev, + const struct ubcore_seg_cfg *cfg, + struct ubcore_udata *udata); +int udma_unregister_seg(struct ubcore_target_seg *seg); +uint64_t key_to_hw_index(uint32_t key); + +#endif /* _UDMA_SEGMENT_H */ -- Gitee From 304d20fbdad05034d0a6398b4327c057a7596728 Mon Sep 17 00:00:00 2001 From: Weibo Zhao Date: Mon, 21 Aug 2023 19:31:58 +0800 Subject: [PATCH 18/42] hns3 udma: support import and unimport segment driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA ----------------------------------------------------------- The mainly part of function of import segment is excuted by urma. Driver should only alloc and free memory. Signed-off-by: Weibo Zhao --- drivers/ub/hw/hns3/hns3_udma_main.c | 2 ++ drivers/ub/hw/hns3/hns3_udma_segment.c | 20 ++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_segment.h | 4 ++++ 3 files changed, 26 insertions(+) diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c index 77d6979e4197..b7ea82c37cb5 100644 --- a/drivers/ub/hw/hns3/hns3_udma_main.c +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -182,6 +182,8 @@ static struct ubcore_ops g_udma_dev_ops = { .mmap = udma_mmap, .register_seg = udma_register_seg, .unregister_seg = udma_unregister_seg, + .import_seg = udma_import_seg, + .unimport_seg = udma_unimport_seg, .create_jfc = udma_create_jfc, .modify_jfc = udma_modify_jfc, .destroy_jfc = udma_destroy_jfc, diff --git a/drivers/ub/hw/hns3/hns3_udma_segment.c b/drivers/ub/hw/hns3/hns3_udma_segment.c index e4a723d585a5..acb9c2ee9715 100644 --- a/drivers/ub/hw/hns3/hns3_udma_segment.c +++ b/drivers/ub/hw/hns3/hns3_udma_segment.c @@ -311,3 +311,23 @@ int udma_unregister_seg(struct ubcore_target_seg *seg) return 0; } + +struct ubcore_target_seg *udma_import_seg(struct ubcore_device *dev, + const struct ubcore_target_seg_cfg *cfg, + struct ubcore_udata *udata) +{ + struct ubcore_target_seg *tseg; + + tseg = kcalloc(1, sizeof(*tseg), GFP_KERNEL); + if (!tseg) + return NULL; + + return tseg; +} + +int udma_unimport_seg(struct ubcore_target_seg *tseg) +{ + kfree(tseg); + + return 0; +} diff --git a/drivers/ub/hw/hns3/hns3_udma_segment.h b/drivers/ub/hw/hns3/hns3_udma_segment.h index 37eb67d5f795..c97ab98b57dd 100644 --- a/drivers/ub/hw/hns3/hns3_udma_segment.h +++ b/drivers/ub/hw/hns3/hns3_udma_segment.h @@ -79,6 +79,10 @@ struct ubcore_target_seg *udma_register_seg(struct ubcore_device *dev, const struct ubcore_seg_cfg *cfg, struct ubcore_udata *udata); int udma_unregister_seg(struct ubcore_target_seg *seg); +struct ubcore_target_seg *udma_import_seg(struct ubcore_device *dev, + const struct ubcore_target_seg_cfg *cfg, + struct ubcore_udata *udata); +int udma_unimport_seg(struct ubcore_target_seg *tseg); uint64_t key_to_hw_index(uint32_t key); #endif /* _UDMA_SEGMENT_H */ -- Gitee From 3ef7a94b57199376edb1b36172cb4389c966ce9a Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Tue, 22 Aug 2023 15:41:42 +0800 Subject: [PATCH 19/42] hns3 udma: support 1-bit ECC recover function. driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA --------------------------------------------------------------------- A pulse signal is generated when a 1-bit ECC error occurs on the SRAM of UDMA hardware. After receiving a 1-bit ECC abnormal interrupt, UDMA driver assists 1-bit ECC recovery with CMQ commands. Signed-off-by: Yu Zhang --- drivers/ub/hw/hns3/hns3_udma_device.h | 1 + drivers/ub/hw/hns3/hns3_udma_eq.c | 157 +++++++++++++++++++++++++- drivers/ub/hw/hns3/hns3_udma_eq.h | 42 +++++++ drivers/ub/hw/hns3/hns3_udma_hw.h | 5 + 4 files changed, 202 insertions(+), 3 deletions(-) diff --git a/drivers/ub/hw/hns3/hns3_udma_device.h b/drivers/ub/hw/hns3/hns3_udma_device.h index f12eb0c2985a..fe689cc84d1c 100644 --- a/drivers/ub/hw/hns3/hns3_udma_device.h +++ b/drivers/ub/hw/hns3/hns3_udma_device.h @@ -698,6 +698,7 @@ struct udma_dev { const struct udma_hw *hw; void *priv; struct workqueue_struct *irq_workq; + struct work_struct ecc_work; uint16_t func_id; uint32_t func_num; uint32_t cong_algo_tmpl_id; diff --git a/drivers/ub/hw/hns3/hns3_udma_eq.c b/drivers/ub/hw/hns3/hns3_udma_eq.c index a1beecbd02a4..9fcd321e9ccb 100644 --- a/drivers/ub/hw/hns3/hns3_udma_eq.c +++ b/drivers/ub/hw/hns3/hns3_udma_eq.c @@ -16,6 +16,7 @@ #include #include "hnae3.h" #include "hns3_udma_hem.h" +#include "hns3_udma_device.h" #include "hns3_udma_jfc.h" #include "hns3_udma_jfr.h" #include "hns3_udma_qp.h" @@ -249,8 +250,7 @@ static void udma_init_irq_work(struct udma_dev *udma_dev, struct udma_eq *eq, { struct udma_work *irq_work; - irq_work = kzalloc(sizeof(struct udma_work), - GFP_ATOMIC); + irq_work = kzalloc(sizeof(struct udma_work), GFP_ATOMIC); if (!irq_work) return; @@ -391,6 +391,154 @@ static int udma_ceq_int(struct udma_dev *udma_dev, return ceqe_found; } +static int fmea_ram_ecc_query(struct udma_dev *udma_dev, + struct fmea_ram_ecc *ecc_info) +{ + struct udma_cmq_desc desc; + struct udma_cmq_req *req = (struct udma_cmq_req *)desc.data; + int ret; + + udma_cmq_setup_basic_desc(&desc, UDMA_QUERY_RAM_ECC, true); + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) + return ret; + + ecc_info->is_ecc_err = udma_reg_read(req, QUERY_RAM_ECC_1BIT_ERR); + ecc_info->res_type = udma_reg_read(req, QUERY_RAM_ECC_RES_TYPE); + ecc_info->index = udma_reg_read(req, QUERY_RAM_ECC_TAG); + + return 0; +} + +static int fmea_recover_gmv(struct udma_dev *udma_dev, uint32_t idx) +{ + struct udma_cmq_desc desc; + struct udma_cmq_req *req = (struct udma_cmq_req *)desc.data; + uint32_t addr_upper; + uint32_t addr_low; + int ret; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_CFG_GMV_BT, true); + udma_reg_write(req, CFG_GMV_BT_IDX, idx); + udma_reg_write(req, CFG_GMV_BT_VF_ID, 0); + + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) { + dev_err(udma_dev->dev, + "failed to execute cmd to read gmv, ret = %d.\n", ret); + return ret; + } + + addr_low = udma_reg_read(req, CFG_GMV_BT_BA_L); + addr_upper = udma_reg_read(req, CFG_GMV_BT_BA_H); + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_CFG_GMV_BT, false); + udma_reg_write(req, CFG_GMV_BT_BA_L, addr_low); + udma_reg_write(req, CFG_GMV_BT_BA_H, addr_upper); + udma_reg_write(req, CFG_GMV_BT_IDX, idx); + udma_reg_write(req, CFG_GMV_BT_VF_ID, 0); + + return udma_cmq_send(udma_dev, &desc, 1); +} + +static uint64_t fmea_get_ram_res_addr(uint32_t res_type, uint64_t *data) +{ + if (res_type == ECC_RESOURCE_QPC_TIMER || + res_type == ECC_RESOURCE_CQC_TIMER || + res_type == ECC_RESOURCE_SCCC) + return le64_to_cpu(*data); + + return le64_to_cpu(*data) << PAGE_SHIFT; +} + +static int fmea_recover_others(struct udma_dev *udma_dev, uint32_t res_type, + uint32_t index) +{ + struct udma_cmd_mailbox *mailbox = udma_alloc_cmd_mailbox(udma_dev); + uint8_t write_bt0_op = fmea_ram_res[res_type].write_bt0_op; + uint8_t read_bt0_op = fmea_ram_res[res_type].read_bt0_op; + struct udma_cmq_desc desc; + struct udma_mbox *mb; + uint64_t addr; + int ret; + + if (IS_ERR_OR_NULL(mailbox)) + return PTR_ERR(mailbox); + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mbox_desc_init(mb, 0, mailbox->dma, index, read_bt0_op); + ret = udma_cmd_mbox(udma_dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); + if (ret) { + dev_err(udma_dev->dev, + "failed to execute cmd to read fmea ram, ret = %d.\n", + ret); + goto out; + } + + addr = fmea_get_ram_res_addr(res_type, (uint64_t *)(mailbox->buf)); + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mbox_desc_init(mb, addr, 0, index, write_bt0_op); + ret = udma_cmd_mbox(udma_dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); + if (ret) { + dev_err(udma_dev->dev, + "failed to execute cmd to write fmea ram, ret = %d.\n", + ret); + } + +out: + udma_free_cmd_mailbox(udma_dev, mailbox); + + return ret; +} + +static void fmea_ram_ecc_recover(struct udma_dev *udma_dev, + struct fmea_ram_ecc *ecc_info) +{ + uint32_t res_type = ecc_info->res_type; + uint32_t index = ecc_info->index; + int ret; + + BUILD_BUG_ON(ARRAY_SIZE(fmea_ram_res) != ECC_RESOURCE_COUNT); + + if (res_type >= ECC_RESOURCE_COUNT) { + dev_err(udma_dev->dev, "unsupported fmea ram ecc type %u.\n", + res_type); + return; + } + + if (res_type == ECC_RESOURCE_GMV) + ret = fmea_recover_gmv(udma_dev, index); + else + ret = fmea_recover_others(udma_dev, res_type, index); + + if (ret) + dev_err(udma_dev->dev, + "failed to recover %s, index = %u, ret = %d.\n", + fmea_ram_res[res_type].name, index, ret); +} + +static void fmea_ram_ecc_work(struct work_struct *ecc_work) +{ + struct udma_dev *udma_dev = + container_of(ecc_work, struct udma_dev, ecc_work); + struct fmea_ram_ecc ecc_info = {}; + + if (fmea_ram_ecc_query(udma_dev, &ecc_info)) { + dev_err(udma_dev->dev, "failed to query fmea ram ecc.\n"); + return; + } + + if (!ecc_info.is_ecc_err) { + dev_err(udma_dev->dev, "there is no fmea ram ecc found.\n"); + return; + } + + fmea_ram_ecc_recover(udma_dev, &ecc_info); +} + static irqreturn_t abnormal_interrupt_basic(struct udma_dev *udma_dev, uint32_t int_st) { @@ -435,7 +583,8 @@ static irqreturn_t udma_msix_interrupt_abn(int irq, void *dev_id) if (int_st) { int_work = abnormal_interrupt_basic(udma_dev, int_st); } else { - dev_err(udma_dev->dev, "ECC ERROR not supported yet\n"); + dev_err(udma_dev->dev, "ECC 1bit ERROR!\n"); + queue_work(udma_dev->irq_workq, &udma_dev->ecc_work); int_work = IRQ_HANDLED; } @@ -717,6 +866,8 @@ int udma_init_eq_table(struct udma_dev *udma_dev) if (ret) goto err_create_eq_fail; + INIT_WORK(&udma_dev->ecc_work, fmea_ram_ecc_work); + udma_dev->irq_workq = alloc_ordered_workqueue("udma_irq_workq", 0); if (!udma_dev->irq_workq) { dev_err(dev, "failed to create irq workqueue.\n"); diff --git a/drivers/ub/hw/hns3/hns3_udma_eq.h b/drivers/ub/hw/hns3/hns3_udma_eq.h index b32a6b2e9f27..bafd446fab34 100644 --- a/drivers/ub/hw/hns3/hns3_udma_eq.h +++ b/drivers/ub/hw/hns3/hns3_udma_eq.h @@ -84,6 +84,48 @@ struct udma_eq_db { uint32_t data[2]; }; +struct fmea_ram_ecc { + uint32_t is_ecc_err; + uint32_t res_type; + uint32_t index; +}; + +enum ecc_resource_type { + ECC_RESOURCE_QPC, + ECC_RESOURCE_CQC, + ECC_RESOURCE_MPT, + ECC_RESOURCE_SRQC, + ECC_RESOURCE_GMV, + ECC_RESOURCE_QPC_TIMER, + ECC_RESOURCE_CQC_TIMER, + ECC_RESOURCE_SCCC, + ECC_RESOURCE_COUNT, +}; + +static const struct { + const char *name; + uint8_t read_bt0_op; + uint8_t write_bt0_op; +} fmea_ram_res[] = { + { "ECC_RESOURCE_QPC", + UDMA_CMD_READ_QPC_BT0, UDMA_CMD_WRITE_QPC_BT0 }, + { "ECC_RESOURCE_CQC", + UDMA_CMD_READ_CQC_BT0, UDMA_CMD_WRITE_CQC_BT0 }, + { "ECC_RESOURCE_MPT", + UDMA_CMD_READ_MPT_BT0, UDMA_CMD_WRITE_MPT_BT0 }, + { "ECC_RESOURCE_SRQC", + UDMA_CMD_READ_SRQC_BT0, UDMA_CMD_WRITE_SRQC_BT0 }, + /* ECC_RESOURCE_GMV is handled by cmdq, not mailbox */ + { "ECC_RESOURCE_GMV", + 0, 0 }, + { "ECC_RESOURCE_QPC_TIMER", + UDMA_CMD_READ_QPC_TIMER_BT0, UDMA_CMD_WRITE_QPC_TIMER_BT0 }, + { "ECC_RESOURCE_CQC_TIMER", + UDMA_CMD_READ_CQC_TIMER_BT0, UDMA_CMD_WRITE_CQC_TIMER_BT0 }, + { "ECC_RESOURCE_SCCC", + UDMA_CMD_READ_SCCC_BT0, UDMA_CMD_WRITE_SCCC_BT0 }, +}; + #define EQ_DB_FIELD_LOC(h, l) ((uint64_t)(h) << 32 | (l)) #define UDMA_EQ_DB_TAG EQ_DB_FIELD_LOC(7, 0) diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.h b/drivers/ub/hw/hns3/hns3_udma_hw.h index f36116e22b18..da6b0ddb4062 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hw.h +++ b/drivers/ub/hw/hns3/hns3_udma_hw.h @@ -166,6 +166,10 @@ struct udma_query_oor_cmq { #define PF_TIMER_RES_QPC_ITEM_NUM CMQ_REQ_FIELD_LOC(60, 48) #define PF_TIMER_RES_CQC_ITEM_NUM CMQ_REQ_FIELD_LOC(91, 80) +/* Fields of UDMA_QUERY_RAM_ECC */ +#define QUERY_RAM_ECC_1BIT_ERR CMQ_REQ_FIELD_LOC(31, 0) +#define QUERY_RAM_ECC_RES_TYPE CMQ_REQ_FIELD_LOC(63, 32) +#define QUERY_RAM_ECC_TAG CMQ_REQ_FIELD_LOC(95, 64) enum { UDMA_CMD_FLAG_IN = BIT(0), @@ -200,6 +204,7 @@ enum udma_opcode_type { UDMA_OPC_QUERY_VF_RES = 0x850e, UDMA_OPC_CFG_GMV_BT = 0x8510, UDMA_OPC_EXT_CFG = 0x8512, + UDMA_QUERY_RAM_ECC = 0x8513, UDMA_SWITCH_PARAMETER_CFG = 0x1033, UDMA_QUERY_OOR_CAPS = 0xA002, UDMA_OPC_DEID_TBL_ADD = 0xA110, -- Gitee From 5f85590f19c32503b36bdeadb4e0552121e7d9d1 Mon Sep 17 00:00:00 2001 From: Weibo Zhao Date: Tue, 22 Aug 2023 16:42:08 +0800 Subject: [PATCH 20/42] hns3 udma: add support of mmap direct wqe page driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA ---------------------------------------------- Direct wqe address start at bar45 and each qp mmap 65536B of memory. Signed-off-by: Weibo Zhao --- drivers/ub/hw/hns3/hns3_udma_abi.h | 4 ++++ drivers/ub/hw/hns3/hns3_udma_main.c | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/drivers/ub/hw/hns3/hns3_udma_abi.h b/drivers/ub/hw/hns3/hns3_udma_abi.h index a7d0a18ebaab..4b8942804aba 100644 --- a/drivers/ub/hw/hns3/hns3_udma_abi.h +++ b/drivers/ub/hw/hns3/hns3_udma_abi.h @@ -19,6 +19,9 @@ #include #define MAP_COMMAND_MASK 0xff +#define MAP_INDEX_MASK 0xffffff +#define MAP_INDEX_SHIFT 8 +#define UDMA_DWQE_PAGE_SIZE 65536 #define UDMA_JETTY_X_PREFIX_BIT_NUM 2 #define UDMA_JFR_QPN_PREFIX 0x1 #define UDMA_ADDR_4K_MASK 0xfffUL @@ -26,6 +29,7 @@ enum { UDMA_MMAP_UAR_PAGE, + UDMA_MMAP_DWQE_PAGE, }; enum udma_jfc_init_attr_mask { diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c index b7ea82c37cb5..7c14bb80488f 100644 --- a/drivers/ub/hw/hns3/hns3_udma_main.c +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -145,9 +145,16 @@ static int get_mmap_cmd(struct vm_area_struct *vma) return (vma->vm_pgoff & MAP_COMMAND_MASK); } +static uint64_t get_mmap_idx(struct vm_area_struct *vma) +{ + return ((vma->vm_pgoff >> MAP_INDEX_SHIFT) & MAP_INDEX_MASK); +} + static int udma_mmap(struct ubcore_ucontext *uctx, struct vm_area_struct *vma) { struct udma_dev *udma_dev = to_udma_dev(uctx->ub_dev); + uint64_t address; + uint64_t qpn; int cmd; if (((vma->vm_end - vma->vm_start) % PAGE_SIZE) != 0) { @@ -165,6 +172,15 @@ static int udma_mmap(struct ubcore_ucontext *uctx, struct vm_area_struct *vma) PAGE_SIZE, vma->vm_page_prot)) return -EAGAIN; break; + case UDMA_MMAP_DWQE_PAGE: + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + qpn = get_mmap_idx(vma); + address = udma_dev->dwqe_page + qpn * UDMA_DWQE_PAGE_SIZE; + if (io_remap_pfn_range(vma, vma->vm_start, + address >> PAGE_SHIFT, + UDMA_DWQE_PAGE_SIZE, vma->vm_page_prot)) + return -EAGAIN; + break; default: dev_err(udma_dev->dev, "mmap failed, cmd(%d) not support\n", cmd); -- Gitee From 203ec15512ff5494b1453860b4bc25a64fdda294 Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Tue, 22 Aug 2023 17:34:52 +0800 Subject: [PATCH 21/42] hns3 udma: support modify jfr function driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA --------------------------------------------------------------------- The modify_jfr function is used to modify the threshold of the remaining WQEs in the JFR. When the number of remaining WQEs is less than the threshold, the UDMA hardware triggers the JFR_LIMIT_REACH asynchronous event. Signed-off-by: Yu Zhang --- drivers/ub/hw/hns3/hns3_udma_jfr.c | 61 +++++++++++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_jfr.h | 2 + drivers/ub/hw/hns3/hns3_udma_main.c | 1 + 3 files changed, 64 insertions(+) diff --git a/drivers/ub/hw/hns3/hns3_udma_jfr.c b/drivers/ub/hw/hns3/hns3_udma_jfr.c index bdaee22485a3..2c3a39a44f73 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jfr.c +++ b/drivers/ub/hw/hns3/hns3_udma_jfr.c @@ -442,6 +442,67 @@ int udma_destroy_jfr(struct ubcore_jfr *jfr) return 0; } +static int udma_hw_modify_srq(struct udma_dev *dev, uint32_t jfrn, + uint16_t jfr_limit) +{ + struct udma_jfr_context *jfr_context; + struct udma_jfr_context *jfrc_mask; + struct udma_cmd_mailbox *mailbox; + struct udma_cmq_desc desc; + struct udma_mbox *mb; + int ret; + + mailbox = udma_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + jfr_context = (struct udma_jfr_context *)mailbox->buf; + jfrc_mask = (struct udma_jfr_context *)mailbox->buf + 1; + memset(jfrc_mask, 0xff, sizeof(*jfrc_mask)); + udma_reg_write(jfr_context, SRQC_LIMIT_WL, jfr_limit); + udma_reg_clear(jfrc_mask, SRQC_LIMIT_WL); + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mbox_desc_init(mb, mailbox->dma, 0, jfrn, UDMA_CMD_MODIFY_SRQC); + + ret = udma_cmd_mbox(dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); + if (ret) + dev_err(dev->dev, "modify JFR(%u) cmd error(%d).\n", + jfrn, ret); + udma_free_cmd_mailbox(dev, mailbox); + + return ret; +} + +int udma_modify_jfr(struct ubcore_jfr *jfr, const struct ubcore_jfr_attr *attr, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_dev = to_udma_dev(jfr->ub_dev); + struct udma_jfr *udma_jfr = to_udma_jfr(jfr); + uint32_t jfr_limit; + int ret; + + if (!(attr->mask & UBCORE_JFR_RX_THRESHOLD)) { + dev_err(udma_dev->dev, "JFR threshold mask is not set.\n"); + return -EINVAL; + } + + jfr_limit = attr->rx_threshold; + if (jfr_limit > udma_jfr->wqe_cnt) { + dev_err(udma_dev->dev, + "JFR limit(%u) larger than wqe num(%u).\n", + jfr_limit, udma_jfr->wqe_cnt); + return -EINVAL; + } + + ret = udma_hw_modify_srq(udma_dev, udma_jfr->jfrn, jfr_limit); + if (ret) + dev_err(udma_dev->dev, + "hw modify srq failed, ret = %d.\n", ret); + + return ret; +} + void udma_jfr_event(struct udma_dev *udma_dev, uint32_t jfrn, int event_type) { struct udma_jfr_table *jfr_table = &udma_dev->jfr_table; diff --git a/drivers/ub/hw/hns3/hns3_udma_jfr.h b/drivers/ub/hw/hns3/hns3_udma_jfr.h index 3757aac10cb7..fb234e6c00c6 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jfr.h +++ b/drivers/ub/hw/hns3/hns3_udma_jfr.h @@ -87,6 +87,8 @@ static inline struct udma_jfr *to_udma_jfr(struct ubcore_jfr *ubcore_jfr) struct ubcore_jfr *udma_create_jfr(struct ubcore_device *dev, const struct ubcore_jfr_cfg *cfg, struct ubcore_udata *udata); int udma_destroy_jfr(struct ubcore_jfr *jfr); +int udma_modify_jfr(struct ubcore_jfr *jfr, const struct ubcore_jfr_attr *attr, + struct ubcore_udata *udata); void udma_jfr_event(struct udma_dev *udma_dev, uint32_t jfrn, int event_type); #endif /* _UDMA_JFR_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c index 7c14bb80488f..e72c77724a94 100644 --- a/drivers/ub/hw/hns3/hns3_udma_main.c +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -204,6 +204,7 @@ static struct ubcore_ops g_udma_dev_ops = { .modify_jfc = udma_modify_jfc, .destroy_jfc = udma_destroy_jfc, .create_jfr = udma_create_jfr, + .modify_jfr = udma_modify_jfr, .destroy_jfr = udma_destroy_jfr, }; -- Gitee From 0cc58f64b8c08f3fddfe802742823a716fe79783 Mon Sep 17 00:00:00 2001 From: Weibo Zhao Date: Wed, 23 Aug 2023 14:19:57 +0800 Subject: [PATCH 22/42] hns3 udma: add support of query_device_attr driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA ---------------------------------------------------------- Query_device_attr is a function that can query the abilities of driver and hardware such as max number of jetty. Signed-off-by: Weibo Zhao --- drivers/ub/hw/hns3/hns3_udma_device.h | 36 ++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_main.c | 61 +++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) diff --git a/drivers/ub/hw/hns3/hns3_udma_device.h b/drivers/ub/hw/hns3/hns3_udma_device.h index fe689cc84d1c..4132fe8316b0 100644 --- a/drivers/ub/hw/hns3/hns3_udma_device.h +++ b/drivers/ub/hw/hns3/hns3_udma_device.h @@ -65,6 +65,8 @@ #define UDMA_TX_CMQ_PI_REG 0x07010 #define UDMA_TX_CMQ_CI_REG 0x07014 +#define UDMA_MAX_MSG_LEN 0x80000000 + #define UDMA_MAX_BT_REGION 3 #define UDMA_MAX_BT_LEVEL 3 @@ -123,6 +125,11 @@ #define UDMA_VF_ABN_INT_EN_REG 0x13008 #define UDMA_VF_EVENT_INT_EN_REG 0x1300c #define EQ_REG_OFFSET 0x4 +#define MTU_VAL_256 256 +#define MTU_VAL_512 512 +#define MTU_VAL_1024 1024 +#define MTU_VAL_2048 2048 +#define MTU_VAL_4096 4096 #define UDMA_DEFAULT_MAX_JETTY_X_SHIFT 8 #define UDMA_DB_ADDR_OFFSET 0x230 @@ -188,6 +195,21 @@ enum { TYPE_CSQ = 1 }; +enum udma_cong_type { + UDMA_CONG_TYPE_DCQCN, + UDMA_CONG_TYPE_LDCP, + UDMA_CONG_TYPE_HC3, + UDMA_CONG_TYPE_DIP, + UDMA_CONG_TYPE_TOTAL, +}; + +enum udma_cong_sel { + UDMA_CONG_SEL_DCQCN = 1 << UDMA_CONG_TYPE_DCQCN, + UDMA_CONG_SEL_LDCP = 1 << UDMA_CONG_TYPE_LDCP, + UDMA_CONG_SEL_HC3 = 1 << UDMA_CONG_TYPE_HC3, + UDMA_CONG_SEL_DIP = 1 << UDMA_CONG_TYPE_DIP, +}; + enum udma_qp_state { QPS_RESET, QPS_RTR = 2, @@ -201,6 +223,20 @@ enum { UDMA_BUF_NOFAIL = BIT(2), }; +static inline enum ubcore_mtu udma_mtu_int_to_enum(int mtu) +{ + if (mtu >= MTU_VAL_4096) + return UBCORE_MTU_4096; + else if (mtu >= MTU_VAL_2048) + return UBCORE_MTU_2048; + else if (mtu >= MTU_VAL_1024) + return UBCORE_MTU_1024; + else if (mtu >= MTU_VAL_512) + return UBCORE_MTU_512; + else + return UBCORE_MTU_256; +} + struct udma_uar { uint64_t pfn; uint64_t index; diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c index e72c77724a94..a2cdb213b13a 100644 --- a/drivers/ub/hw/hns3/hns3_udma_main.c +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -17,6 +17,7 @@ #include #include "urma/ubcore_api.h" #include "hns3_udma_abi.h" +#include "hnae3.h" #include "hns3_udma_device.h" #include "hns3_udma_hem.h" #include "hns3_udma_jfr.h" @@ -189,10 +190,70 @@ static int udma_mmap(struct ubcore_ucontext *uctx, struct vm_area_struct *vma) return 0; } +static uint16_t query_congest_alg(uint8_t udma_cc_caps) +{ + uint16_t ubcore_cc_alg = 0; + + if (udma_cc_caps & UDMA_CONG_SEL_DCQCN) + ubcore_cc_alg |= UBCORE_CC_DCQCN; + if (udma_cc_caps & UDMA_CONG_SEL_LDCP) + ubcore_cc_alg |= UBCORE_CC_LDCP; + if (udma_cc_caps & UDMA_CONG_SEL_HC3) + ubcore_cc_alg |= UBCORE_CC_HC3; + if (udma_cc_caps & UDMA_CONG_SEL_DIP) + ubcore_cc_alg |= UBCORE_CC_DIP; + + return ubcore_cc_alg; +} + +static int udma_query_device_attr(struct ubcore_device *dev, + struct ubcore_device_attr *attr) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + struct device *dev_of_udma = udma_dev->dev; + struct net_device *net_dev; + int i; + + attr->guid = udma_dev->sys_image_guid; + attr->dev_cap.max_jfc = (1 << udma_dev->caps.num_jfc_shift); + attr->dev_cap.max_jfs = (1 << udma_dev->caps.num_jfs_shift); + attr->dev_cap.max_jfr = (1 << udma_dev->caps.num_jfr_shift); + attr->dev_cap.max_jetty = (1 << udma_dev->caps.num_jetty_shift); + attr->dev_cap.max_jfc_depth = udma_dev->caps.max_cqes; + attr->dev_cap.max_jfs_depth = udma_dev->caps.max_wqes; + attr->dev_cap.max_jfr_depth = udma_dev->caps.max_srq_wrs; + attr->dev_cap.max_jfs_inline_size = udma_dev->caps.max_sq_inline; + attr->dev_cap.max_jfs_sge = udma_dev->caps.max_sq_sg; + attr->dev_cap.max_jfr_sge = udma_dev->caps.max_srq_sges; + attr->dev_cap.max_msg_size = UDMA_MAX_MSG_LEN; + attr->dev_cap.trans_mode = UBCORE_TP_RM | UBCORE_TP_UM; + attr->dev_cap.feature.bs.oor = udma_dev->caps.oor_en; + attr->port_cnt = udma_dev->caps.num_ports; + attr->dev_cap.comp_vector_cnt = udma_dev->caps.num_comp_vectors; + attr->vf_cnt = udma_dev->func_num - 1; + attr->dev_cap.feature.bs.jfc_inline = !!(udma_dev->caps.flags & UDMA_CAP_FLAG_CQE_INLINE); + attr->dev_cap.feature.bs.spray_en = 1; + attr->dev_cap.max_jfs_rsge = udma_dev->caps.max_sq_sg; + attr->dev_cap.congestion_ctrl_alg = query_congest_alg(udma_dev->caps.cong_type); + + for (i = 0; i < udma_dev->caps.num_ports; i++) { + net_dev = udma_dev->uboe.netdevs[i]; + if (!net_dev) { + dev_err(dev_of_udma, "Find netdev %u failed!\n", i); + return -EINVAL; + } + attr->port_attr[i].max_mtu = + udma_mtu_int_to_enum(net_dev->max_mtu); + } + + return 0; +} + static struct ubcore_ops g_udma_dev_ops = { .owner = THIS_MODULE, .abi_version = 1, .set_eid = udma_set_eid, + .query_device_attr = udma_query_device_attr, .alloc_ucontext = udma_alloc_ucontext, .free_ucontext = udma_free_ucontext, .mmap = udma_mmap, -- Gitee From 563866c1473b31ec601c5118ce0cb9dceab8ad6f Mon Sep 17 00:00:00 2001 From: Weibo Zhao Date: Wed, 23 Aug 2023 15:38:35 +0800 Subject: [PATCH 23/42] hns3 udma: add support of query_device_status driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA ------------------------------------------------------ Query_device_status is a function that can query device and port's status such as mtu and speed and if is active. Signed-off-by: Weibo Zhao --- drivers/ub/hw/hns3/hns3_udma_device.h | 1 + drivers/ub/hw/hns3/hns3_udma_hw.c | 20 ++++++++++ drivers/ub/hw/hns3/hns3_udma_hw.h | 11 ++++++ drivers/ub/hw/hns3/hns3_udma_main.c | 56 +++++++++++++++++++++++++++ 4 files changed, 88 insertions(+) diff --git a/drivers/ub/hw/hns3/hns3_udma_device.h b/drivers/ub/hw/hns3/hns3_udma_device.h index 4132fe8316b0..4de03c570371 100644 --- a/drivers/ub/hw/hns3/hns3_udma_device.h +++ b/drivers/ub/hw/hns3/hns3_udma_device.h @@ -625,6 +625,7 @@ struct udma_caps { uint32_t num_jfr_shift; uint32_t num_jetty_shift; uint8_t poe_ch_num; + uint32_t speed; }; struct udma_idx_table { diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.c b/drivers/ub/hw/hns3/hns3_udma_hw.c index 72a93dfc189f..6e82a67998d7 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hw.c +++ b/drivers/ub/hw/hns3/hns3_udma_hw.c @@ -175,6 +175,24 @@ static void set_default_jetty_caps(struct udma_dev *dev) caps->num_jetty_shift = UDMA_DEFAULT_MAX_JETTY_X_SHIFT; } +static void query_hw_speed(struct udma_dev *udma_dev) +{ + struct udma_port_info_cmq *resp; + struct udma_cmq_desc desc; + int ret; + + resp = (struct udma_port_info_cmq *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_QUERY_PORT_INFO, true); + resp->query_type = UDMA_QUERY_PORT_INFO; + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) { + dev_err(udma_dev->dev, "failed to query speed, ret = %d. set default 100G\n", ret); + udma_dev->caps.speed = SPEED_100G; + return; + } + udma_dev->caps.speed = resp->speed; +} + static int udma_query_caps(struct udma_dev *udma_dev) { enum udma_opcode_type opcode = UDMA_OPC_QUERY_PF_CAPS_NUM; @@ -351,6 +369,8 @@ static int udma_query_caps(struct udma_dev *udma_dev) QUERY_PF_CAPS_D_RQWQE_HOP_NUM_S); set_default_jetty_caps(udma_dev); + query_hw_speed(udma_dev); + return 0; } diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.h b/drivers/ub/hw/hns3/hns3_udma_hw.h index da6b0ddb4062..e6beb8142205 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hw.h +++ b/drivers/ub/hw/hns3/hns3_udma_hw.h @@ -113,6 +113,10 @@ struct udma_query_oor_cmq { #define UDMA_BA_PG_SZ_SUPPORTED_256K 6 #define UDMA_BA_PG_SZ_SUPPORTED_16K 2 +#define UDMA_QUERY_PORT_INFO 1 +#define SPEED_100G 100000 +#define SPEED_200G 200000 + /* Fields of UDMA_OPC_EXT_CFG */ #define EXT_CFG_VF_ID CMQ_REQ_FIELD_LOC(31, 0) #define EXT_CFG_QP_PI_INDEX CMQ_REQ_FIELD_LOC(45, 32) @@ -209,6 +213,7 @@ enum udma_opcode_type { UDMA_QUERY_OOR_CAPS = 0xA002, UDMA_OPC_DEID_TBL_ADD = 0xA110, UDMA_OPC_CFG_GMV_TBL = 0xA140, + UDMA_OPC_QUERY_PORT_INFO = 0x7104, }; #define UDMA_QUERY_PF_CAPS_CMD_NUM 5 @@ -461,4 +466,10 @@ union udma_eid { } bit32_data; }; +struct udma_port_info_cmq { + uint32_t speed; + uint8_t query_type; + uint8_t rsv[19]; +}; + #endif /* _UDMA_HW_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c index a2cdb213b13a..48bf6d62c823 100644 --- a/drivers/ub/hw/hns3/hns3_udma_main.c +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -249,11 +249,67 @@ static int udma_query_device_attr(struct ubcore_device *dev, return 0; } +static int udma_get_active_speed(uint32_t speed, struct ubcore_port_status *port_status) +{ + if (speed == SPEED_100G) { + port_status->active_width = UBCORE_LINK_X1; + port_status->active_speed = UBCORE_SP_100G; + } else if (speed == SPEED_200G) { + port_status->active_width = UBCORE_LINK_X1; + port_status->active_speed = UBCORE_SP_200G; + } else { + return -EINVAL; + } + + return 0; +} + +static int udma_query_device_status(const struct ubcore_device *dev, + struct ubcore_device_status *dev_status) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + enum ubcore_mtu net_dev_mtu; + struct net_device *net_dev; + enum ubcore_mtu mtu; + uint8_t port_num; + int ret; + int i; + + port_num = udma_dev->caps.num_ports; + + for (i = 0; i < port_num; i++) { + net_dev = udma_dev->uboe.netdevs[i]; + if (!net_dev) { + dev_err(udma_dev->dev, "Find netdev %u failed!\n", i); + return -EINVAL; + } + + dev_status->port_status[i].state = + (netif_running(net_dev) && + netif_carrier_ok(net_dev)) ? + UBCORE_PORT_ACTIVE : UBCORE_PORT_DOWN; + net_dev_mtu = ubcore_get_mtu(net_dev->max_mtu); + mtu = ubcore_get_mtu(net_dev->mtu); + + dev_status->port_status[i].active_mtu = (enum ubcore_mtu) + (mtu ? min(net_dev_mtu, mtu) : UBCORE_MTU_256); + + ret = udma_get_active_speed(udma_dev->caps.speed, &dev_status->port_status[i]); + if (ret) { + dev_err(udma_dev->dev, "Port[%u] query speed and width failed!\n", i); + return ret; + } + } + + return 0; +} + static struct ubcore_ops g_udma_dev_ops = { .owner = THIS_MODULE, .abi_version = 1, .set_eid = udma_set_eid, .query_device_attr = udma_query_device_attr, + .query_device_status = udma_query_device_status, .alloc_ucontext = udma_alloc_ucontext, .free_ucontext = udma_free_ucontext, .mmap = udma_mmap, -- Gitee From be8691a5b8832142b0eef32185499f30abe95a76 Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Wed, 23 Aug 2023 17:18:40 +0800 Subject: [PATCH 24/42] hns3 udma: support import and unimport jfr driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA --------------------------------------------------------------------- In the kernel mode import_jfr, UDMA driver only needs to create the target jfr structure and returned it to the URMA. In addition, unimport_jfr just free the target jfr. Signed-off-by: Yu Zhang --- drivers/ub/hw/hns3/hns3_udma_jfr.c | 20 ++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_jfr.h | 4 ++++ drivers/ub/hw/hns3/hns3_udma_main.c | 2 ++ 3 files changed, 26 insertions(+) diff --git a/drivers/ub/hw/hns3/hns3_udma_jfr.c b/drivers/ub/hw/hns3/hns3_udma_jfr.c index 2c3a39a44f73..a4111b5083bb 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jfr.c +++ b/drivers/ub/hw/hns3/hns3_udma_jfr.c @@ -442,6 +442,26 @@ int udma_destroy_jfr(struct ubcore_jfr *jfr) return 0; } +struct ubcore_tjetty *udma_import_jfr(struct ubcore_device *dev, + const struct ubcore_tjetty_cfg *cfg, + struct ubcore_udata *udata) +{ + struct ubcore_tjetty *tjfr; + + tjfr = kcalloc(1, sizeof(*tjfr), GFP_KERNEL); + if (!tjfr) + return NULL; + + return tjfr; +} + +int udma_unimport_jfr(struct ubcore_tjetty *tjfr) +{ + kfree(tjfr); + + return 0; +} + static int udma_hw_modify_srq(struct udma_dev *dev, uint32_t jfrn, uint16_t jfr_limit) { diff --git a/drivers/ub/hw/hns3/hns3_udma_jfr.h b/drivers/ub/hw/hns3/hns3_udma_jfr.h index fb234e6c00c6..838aa46fa759 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jfr.h +++ b/drivers/ub/hw/hns3/hns3_udma_jfr.h @@ -87,6 +87,10 @@ static inline struct udma_jfr *to_udma_jfr(struct ubcore_jfr *ubcore_jfr) struct ubcore_jfr *udma_create_jfr(struct ubcore_device *dev, const struct ubcore_jfr_cfg *cfg, struct ubcore_udata *udata); int udma_destroy_jfr(struct ubcore_jfr *jfr); +struct ubcore_tjetty *udma_import_jfr(struct ubcore_device *dev, + const struct ubcore_tjetty_cfg *cfg, + struct ubcore_udata *udata); +int udma_unimport_jfr(struct ubcore_tjetty *tjfr); int udma_modify_jfr(struct ubcore_jfr *jfr, const struct ubcore_jfr_attr *attr, struct ubcore_udata *udata); void udma_jfr_event(struct udma_dev *udma_dev, uint32_t jfrn, int event_type); diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c index 48bf6d62c823..3216ef0d3f9e 100644 --- a/drivers/ub/hw/hns3/hns3_udma_main.c +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -323,6 +323,8 @@ static struct ubcore_ops g_udma_dev_ops = { .create_jfr = udma_create_jfr, .modify_jfr = udma_modify_jfr, .destroy_jfr = udma_destroy_jfr, + .import_jfr = udma_import_jfr, + .unimport_jfr = udma_unimport_jfr, }; static void udma_cleanup_uar_table(struct udma_dev *dev) -- Gitee From 40b935d3657c6f66d0df9d8cd7e81790a4fa15c1 Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Sat, 26 Aug 2023 18:00:36 +0800 Subject: [PATCH 25/42] hns3 udma: support create tp driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA --------------------------------------------------------------------- TP: Before sending or receiving data, a transport layer connection, also called a transport channel (TP channel), needs to be established. This patch only support create jfr tp. Tp for jfs and jetty will be updated in later patch Signed-off-by: Yu Zhang --- drivers/ub/hw/hns3/Makefile | 2 +- drivers/ub/hw/hns3/hns3_udma_abi.h | 23 ++ drivers/ub/hw/hns3/hns3_udma_device.h | 6 + drivers/ub/hw/hns3/hns3_udma_qp.c | 514 ++++++++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_qp.h | 71 ++++ drivers/ub/hw/hns3/hns3_udma_tp.c | 108 ++++++ drivers/ub/hw/hns3/hns3_udma_tp.h | 32 ++ 7 files changed, 755 insertions(+), 1 deletion(-) create mode 100644 drivers/ub/hw/hns3/hns3_udma_tp.c create mode 100644 drivers/ub/hw/hns3/hns3_udma_tp.h diff --git a/drivers/ub/hw/hns3/Makefile b/drivers/ub/hw/hns3/Makefile index 2ef23a914826..4250bbc0ec93 100644 --- a/drivers/ub/hw/hns3/Makefile +++ b/drivers/ub/hw/hns3/Makefile @@ -10,6 +10,6 @@ ccflags-y += -I$(srctree)/drivers/net/ethernet/hisilicon/hns3/ \ $(MODULE_NAME)-objs := hns3_udma_hw.o hns3_udma_main.o hns3_udma_cmd.o \ hns3_udma_hem.o hns3_udma_qp.o hns3_udma_eq.o \ hns3_udma_db.o hns3_udma_jfc.o hns3_udma_jfr.o \ - hns3_udma_segment.o + hns3_udma_segment.o hns3_udma_tp.o obj-$(CONFIG_UB_UDMA_HNS3) := hns3_udma.o diff --git a/drivers/ub/hw/hns3/hns3_udma_abi.h b/drivers/ub/hw/hns3/hns3_udma_abi.h index 4b8942804aba..ee817b23c0de 100644 --- a/drivers/ub/hw/hns3/hns3_udma_abi.h +++ b/drivers/ub/hw/hns3/hns3_udma_abi.h @@ -82,6 +82,29 @@ struct udma_create_jfc_resp { uint32_t jfc_caps; }; +struct udma_create_tp_ucmd { + bool is_jetty; + union { + uint32_t jfs_id; + uint32_t jetty_id; + } ini_id; + union { + uint32_t jfr_id; + uint32_t jetty_id; + } tgt_id; + /* used for create_qp */ + uint64_t buf_addr; + uint64_t db_addr; + uint64_t sdb_addr; +}; + +struct udma_create_tp_resp { + uint64_t cap_flags; + uint32_t qpn; + uint32_t path_mtu; + uint8_t priority; +}; + struct udma_create_ctx_resp { uint32_t num_comp_vectors; uint32_t num_qps_shift; diff --git a/drivers/ub/hw/hns3/hns3_udma_device.h b/drivers/ub/hw/hns3/hns3_udma_device.h index 4de03c570371..3ba30971133f 100644 --- a/drivers/ub/hw/hns3/hns3_udma_device.h +++ b/drivers/ub/hw/hns3/hns3_udma_device.h @@ -45,6 +45,7 @@ #define UDMA_GMV_ENTRY_SZ 32 #define UDMA_CQ_BANK_NUM 4 +#define UDMA_SGE_SHIFT 4 #define UDMA_SGE_SIZE 16 #define UDMA_IDX_QUE_ENTRY_SZ 4 /* The minimum page size is 4K for hardware */ @@ -210,6 +211,11 @@ enum udma_cong_sel { UDMA_CONG_SEL_DIP = 1 << UDMA_CONG_TYPE_DIP, }; +enum udma_qp_type { + QPT_RC, + QPT_UD = 0x3, +}; + enum udma_qp_state { QPS_RESET, QPS_RTR = 2, diff --git a/drivers/ub/hw/hns3/hns3_udma_qp.c b/drivers/ub/hw/hns3/hns3_udma_qp.c index b8909731fbad..407bddab2db6 100644 --- a/drivers/ub/hw/hns3/hns3_udma_qp.c +++ b/drivers/ub/hw/hns3/hns3_udma_qp.c @@ -13,11 +13,230 @@ * */ +#include #include #include "hns3_udma_abi.h" +#include "hns3_udma_jfr.h" #include "hns3_udma_jfc.h" #include "hns3_udma_hem.h" #include "hns3_udma_qp.h" + +static int udma_alloc_reorder_cq_buf(struct udma_dev *udma_dev, + struct udma_qp_attr *qp_attr) +{ + struct udma_caps *caps = &udma_dev->caps; + int buff_sz; + + buff_sz = (1 << caps->reorder_cq_shift) * caps->cqe_sz; + qp_attr->reorder_cq_size = buff_sz; + qp_attr->reorder_cq_page = dma_alloc_coherent(udma_dev->dev, buff_sz, + &qp_attr->reorder_cq_addr, + GFP_KERNEL); + if (!qp_attr->reorder_cq_page) { + dev_err(udma_dev->dev, "Dma alloc coherent failed\n"); + return -ENOMEM; + } + return 0; +} + +static void udma_free_reorder_cq_buf(struct udma_dev *udma_dev, + struct udma_qp_attr *qp_attr) +{ + if (qp_attr->reorder_cq_page) + dma_free_coherent(udma_dev->dev, qp_attr->reorder_cq_size, + qp_attr->reorder_cq_page, + qp_attr->reorder_cq_addr); +} + +int fill_jfr_qp_attr(struct udma_dev *udma_dev, struct udma_qp_attr *qp_attr, + struct udma_create_tp_ucmd *ucmd) +{ + struct udma_jfr *udma_jfr; + struct ubcore_jfr *jfr; + + udma_jfr = (struct udma_jfr *)xa_load(&udma_dev->jfr_table.xa, + ucmd->tgt_id.jfr_id); + if (IS_ERR_OR_NULL(udma_jfr)) { + dev_err(udma_dev->dev, "failed to find jfr\n"); + return -EINVAL; + } + jfr = &udma_jfr->ubcore_jfr; + qp_attr->jfr = udma_jfr; + qp_attr->recv_jfc = to_udma_jfc(jfr->jfr_cfg.jfc); + qp_attr->uctx = qp_attr->jfr->ubcore_jfr.uctx; + qp_attr->qpn_map = &qp_attr->jfr->qpn_map; + + if (jfr->jfr_cfg.trans_mode == UBCORE_TP_UM) { + dev_err(udma_dev->dev, "jfr tp mode error\n"); + return -EINVAL; + } + + qp_attr->qp_type = QPT_RC; + qp_attr->cap.min_rnr_timer = jfr->jfr_cfg.min_rnr_timer; + + return 0; +} + +int udma_fill_qp_attr(struct udma_dev *udma_dev, struct udma_qp_attr *qp_attr, + const struct ubcore_tp_cfg *cfg, struct ubcore_udata *udata) +{ + bool is_target = udata->uctx == NULL ? true : false; + struct udma_create_tp_ucmd ucmd; + struct udma_ucontext *udma_ctx; + int status = 0; + + if (!udata) + return 0; + + if (!is_target) { + status = copy_from_user(&ucmd, (void *)udata->udrv_data->in_addr, + min(udata->udrv_data->in_len, + (uint32_t)sizeof(ucmd))); + if (status) { + dev_err(udma_dev->dev, "failed to copy create tp ucmd\n"); + return status; + } + } else { + memcpy(&ucmd, (void *)udata->udrv_data->in_addr, + min(udata->udrv_data->in_len, (uint32_t)sizeof(ucmd))); + } + + qp_attr->is_tgt = is_target; + qp_attr->is_jetty = ucmd.is_jetty; + qp_attr->remote_eid = cfg->peer_eid; + qp_attr->local_eid = cfg->local_eid; + udma_ctx = to_udma_ucontext(udata->uctx); + + if (!is_target) { + qp_attr->pdn = udma_ctx->pdn; + /* TODO: fill_jfs_qp_attr */ + return 0; + } + + if (!ucmd.is_jetty) + return fill_jfr_qp_attr(udma_dev, qp_attr, &ucmd); + + return status; +} + +static void set_rq_size(struct udma_qp *qp, struct udma_qp_cap *cap) +{ + /* set rq param to 0 */ + qp->rq.wqe_cnt = 0; + qp->rq.max_gs = 1; + cap->max_recv_wr = 0; + cap->max_recv_sge = 0; +} + +static int set_qp_param(struct udma_dev *udma_dev, struct udma_qp *qp, + struct ubcore_udata *udata, + struct udma_create_tp_ucmd *ucmd) +{ + struct udma_qp_attr *qp_attr = &qp->qp_attr; + struct device *dev = udma_dev->dev; + int ret = 0; + + qp->qp_type = qp_attr->qp_type; + + if (!qp_attr->is_tgt) { + qp->retry_cnt = qp_attr->cap.retry_cnt; + qp->ack_timeout = qp_attr->cap.ack_timeout; + qp->rnr_retry = qp_attr->cap.rnr_retry; + qp->priority = qp_attr->priority; + } + + if (qp_attr->cap.max_inline_data > udma_dev->caps.max_sq_inline) + qp_attr->cap.max_inline_data = udma_dev->caps.max_sq_inline; + + qp->max_inline_data = qp_attr->cap.max_inline_data; + + set_rq_size(qp, &qp_attr->cap); + + if (udata && udata->uctx != NULL) { + ret = copy_from_user(ucmd, (void *)udata->udrv_data->in_addr, + min(udata->udrv_data->in_len, + (uint32_t)sizeof(struct udma_create_tp_ucmd))); + if (ret) { + dev_err(dev, "failed to copy create tp ucmd\n"); + return ret; + } + + } + + return ret; +} + +static uint8_t get_least_load_bankid_for_qp(struct udma_bank *bank) +{ + uint32_t least_load = bank[0].inuse; + uint8_t bankid = 0; + uint32_t bankcnt; + uint8_t i; + + for (i = 1; i < UDMA_QP_BANK_NUM; i++) { + bankcnt = bank[i].inuse; + if (bankcnt < least_load) { + least_load = bankcnt; + bankid = i; + } + } + + return bankid; +} + +static int alloc_qpn_with_bankid(struct udma_bank *bank, uint8_t bankid, + uint64_t *qpn) +{ + int idx; + + idx = ida_alloc_range(&bank->ida, bank->next, bank->max, GFP_KERNEL); + if (idx < 0) { + idx = ida_alloc_range(&bank->ida, bank->min, bank->max, + GFP_KERNEL); + if (idx < 0) + return idx; + } + + bank->next = + ((uint32_t)idx + 1) > bank->max ? bank->min : (uint32_t)idx + 1; + + /* the lower 3 bits is bankid */ + *qpn = (idx << 3) | bankid; + + return 0; +} + +static int alloc_qpn(struct udma_dev *udma_dev, struct udma_qp *qp) +{ + struct udma_qpn_bitmap *qpn_map = qp->qp_attr.qpn_map; + struct device *dev = udma_dev->dev; + uint64_t num = 0; + uint8_t bankid; + int ret; + + if (qpn_map->qpn_shift == 0 || qp->qp_type == QPT_UD) { + qp->qpn = gen_qpn(qpn_map->qpn_prefix, + qpn_map->jid << qpn_map->qpn_shift, 0); + } else { + mutex_lock(&qpn_map->bank_mutex); + bankid = get_least_load_bankid_for_qp(qpn_map->bank); + ret = alloc_qpn_with_bankid(&qpn_map->bank[bankid], bankid, + &num); + if (ret) { + dev_err(dev, "failed to alloc QPN, ret = %d\n", ret); + mutex_unlock(&qpn_map->bank_mutex); + return ret; + } + qpn_map->bank[bankid].inuse++; + mutex_unlock(&qpn_map->bank_mutex); + qp->qpn = gen_qpn(qpn_map->qpn_prefix, + qpn_map->jid << qpn_map->qpn_shift, num); + } + atomic_inc(&qpn_map->ref_num); + + return 0; +} + static void init_qpn_bitmap(struct udma_qpn_bitmap *qpn_map, uint32_t qpn_shift) { int i; @@ -68,12 +287,307 @@ void clean_jetty_x_qpn_bitmap(struct udma_qpn_bitmap *qpn_map) mutex_unlock(&qpn_map->bank_mutex); } + +static int alloc_qpc(struct udma_dev *udma_dev, struct udma_qp *qp) +{ + struct udma_qp_table *qp_table = &udma_dev->qp_table; + struct device *dev = udma_dev->dev; + int ret; + + /* Alloc memory for QPC */ + ret = udma_table_get(udma_dev, &qp_table->qp_table, qp->qpn); + if (ret) { + dev_err(dev, "Failed to get QPC table\n"); + goto err_out; + } + + /* Alloc memory for IRRL */ + ret = udma_table_get(udma_dev, &qp_table->irrl_table, qp->qpn); + if (ret) { + dev_err(dev, "Failed to get IRRL table\n"); + goto err_put_qp; + } + + if (udma_dev->caps.trrl_entry_sz) { + /* Alloc memory for TRRL */ + ret = udma_table_get(udma_dev, &qp_table->trrl_table, + qp->qpn); + if (ret) { + dev_err(dev, "Failed to get TRRL table\n"); + goto err_put_irrl; + } + } + + if (udma_dev->caps.flags & UDMA_CAP_FLAG_QP_FLOW_CTRL) { + /* Alloc memory for SCC CTX */ + ret = udma_table_get(udma_dev, &qp_table->sccc_table, + qp->qpn); + if (ret) { + dev_err(dev, "Failed to get SCC CTX table\n"); + goto err_put_trrl; + } + } + + if (udma_dev->caps.reorder_cq_buffer_en) { + ret = udma_alloc_reorder_cq_buf(udma_dev, &qp->qp_attr); + if (ret) + dev_warn(udma_dev->dev, + "failed to alloc reorder cq buffer.\n"); + } + + return 0; + +err_put_trrl: + if (udma_dev->caps.trrl_entry_sz) + udma_table_put(udma_dev, &qp_table->trrl_table, qp->qpn); +err_put_irrl: + udma_table_put(udma_dev, &qp_table->irrl_table, qp->qpn); +err_put_qp: + udma_table_put(udma_dev, &qp_table->qp_table, qp->qpn); +err_out: + return ret; +} + +static void udma_lock_cqs(struct udma_jfc *send_jfc, struct udma_jfc *recv_jfc) + __acquires(&send_jfc->lock) + __acquires(&recv_jfc->lock) +{ + if (unlikely(send_jfc == NULL && recv_jfc == NULL)) { + __acquire(&send_jfc->lock); + __acquire(&recv_jfc->lock); + } else if (unlikely(send_jfc != NULL && recv_jfc == NULL)) { + spin_lock_irq(&send_jfc->lock); + __acquire(&recv_jfc->lock); + } else if (unlikely(send_jfc == NULL && recv_jfc != NULL)) { + spin_lock_irq(&recv_jfc->lock); + __acquire(&send_jfc->lock); + } else if (send_jfc == recv_jfc) { + spin_lock_irq(&send_jfc->lock); + __acquire(&recv_jfc->lock); + } else if (send_jfc->cqn < recv_jfc->cqn) { + spin_lock_irq(&send_jfc->lock); + spin_lock_nested(&recv_jfc->lock, SINGLE_DEPTH_NESTING); + } else { + spin_lock_irq(&recv_jfc->lock); + spin_lock_nested(&send_jfc->lock, SINGLE_DEPTH_NESTING); + } +} + +static void udma_unlock_cqs(struct udma_jfc *send_jfc, + struct udma_jfc *recv_jfc) + __releases(&send_jfc->lock) + __releases(&recv_jfc->lock) +{ + if (unlikely(send_jfc == NULL && recv_jfc == NULL)) { + __release(&recv_jfc->lock); + __release(&send_jfc->lock); + } else if (unlikely(send_jfc != NULL && recv_jfc == NULL)) { + __release(&recv_jfc->lock); + spin_unlock(&send_jfc->lock); + } else if (unlikely(send_jfc == NULL && recv_jfc != NULL)) { + __release(&send_jfc->lock); + spin_unlock(&recv_jfc->lock); + } else if (send_jfc == recv_jfc) { + __release(&recv_jfc->lock); + spin_unlock_irq(&send_jfc->lock); + } else if (send_jfc->cqn < recv_jfc->cqn) { + spin_unlock(&recv_jfc->lock); + spin_unlock_irq(&send_jfc->lock); + } else { + spin_unlock(&send_jfc->lock); + spin_unlock_irq(&recv_jfc->lock); + } +} + +static void add_qp_to_list(struct udma_dev *udma_dev, struct udma_qp *qp, + struct udma_jfc *send_jfc, struct udma_jfc *recv_jfc) +{ + unsigned long flags; + + spin_lock_irqsave(&udma_dev->qp_list_lock, flags); + udma_lock_cqs(send_jfc, recv_jfc); + + list_add_tail(&qp->node, &udma_dev->qp_list); + + if (send_jfc) + list_add_tail(&qp->sq_node, &send_jfc->sq_list); + if (recv_jfc) + list_add_tail(&qp->rq_node, &recv_jfc->rq_list); + + udma_unlock_cqs(send_jfc, recv_jfc); + spin_unlock_irqrestore(&udma_dev->qp_list_lock, flags); +} + +static int udma_qp_store(struct udma_dev *udma_dev, + struct udma_qp *qp) +{ + struct udma_qp_attr *qp_attr = &qp->qp_attr; + struct xarray *xa = &udma_dev->qp_table.xa; + int ret; + + ret = xa_err(xa_store_irq(xa, qp->qpn, qp, GFP_KERNEL)); + if (ret) + dev_err(udma_dev->dev, "Failed to xa store for QPC\n"); + else + /* add QP to device's QP list for softwc */ + add_qp_to_list(udma_dev, qp, qp_attr->send_jfc, + qp_attr->recv_jfc); + + return ret; +} + +static void udma_qp_remove(struct udma_dev *udma_dev, struct udma_qp *qp) +{ + struct udma_qp_attr *qp_attr = &qp->qp_attr; + struct xarray *xa = &udma_dev->qp_table.xa; + struct udma_jfc *send_jfc; + struct udma_jfc *recv_jfc; + unsigned long flags; + + send_jfc = qp_attr->send_jfc; + recv_jfc = qp_attr->recv_jfc; + + xa_lock_irqsave(xa, flags); + __xa_erase(xa, qp->qpn); + xa_unlock_irqrestore(xa, flags); + + spin_lock_irqsave(&udma_dev->qp_list_lock, flags); + udma_lock_cqs(send_jfc, recv_jfc); + + list_del(&qp->node); + + if (send_jfc) + list_del(&qp->sq_node); + if (recv_jfc) + list_del(&qp->rq_node); + + udma_unlock_cqs(send_jfc, recv_jfc); + spin_unlock_irqrestore(&udma_dev->qp_list_lock, flags); +} + +static void free_qpc(struct udma_dev *udma_dev, struct udma_qp *qp) +{ + struct udma_qp_table *qp_table = &udma_dev->qp_table; + + if (udma_dev->caps.reorder_cq_buffer_en) + udma_free_reorder_cq_buf(udma_dev, &qp->qp_attr); + + if (udma_dev->caps.flags & UDMA_CAP_FLAG_QP_FLOW_CTRL) + udma_table_put(udma_dev, &qp_table->sccc_table, qp->qpn); + + if (udma_dev->caps.trrl_entry_sz) + udma_table_put(udma_dev, &qp_table->trrl_table, qp->qpn); + + udma_table_put(udma_dev, &qp_table->irrl_table, qp->qpn); +} + static inline uint8_t get_qp_bankid(uint64_t qpn) { /* The lower 3 bits of QPN are used to hash to different banks */ return (uint8_t)(qpn & QP_BANKID_MASK); } +static void free_qpn(struct udma_qp *qp) +{ + struct udma_qpn_bitmap *qpn_map = qp->qp_attr.qpn_map; + uint8_t bankid; + + if (qpn_map->qpn_shift == 0 || qp->qp_type == QPT_UD) + return; + + bankid = get_qp_bankid(qp->qpn); + + mutex_lock(&qpn_map->bank_mutex); + if (!ida_is_empty(&qpn_map->bank[bankid].ida)) { + ida_free(&qpn_map->bank[bankid].ida, + (qp->qpn & GENMASK(qpn_map->qpn_shift - 1, 0)) >> + QP_BANKID_SHIFT); + } + qpn_map->bank[bankid].inuse--; + mutex_unlock(&qpn_map->bank_mutex); +} + +int udma_create_qp_common(struct udma_dev *udma_dev, struct udma_qp *qp, + struct ubcore_udata *udata) +{ + struct device *dev = udma_dev->dev; + struct udma_create_tp_ucmd ucmd; + struct udma_create_tp_resp resp; + int ret; + + qp->state = QPS_RESET; + + ret = set_qp_param(udma_dev, qp, udata, &ucmd); + if (ret) { + dev_err(dev, "failed to set QP param, ret = %d.\n", ret); + return ret; + } + + ret = alloc_qpn(udma_dev, qp); + if (ret) { + dev_err(dev, "failed to alloc QPN, ret = %d.\n", ret); + goto err_qpn; + } + + if (!qp->qpn) { + ret = -EINVAL; + goto err_qpn; + } + + ret = alloc_qpc(udma_dev, qp); + if (ret) { + dev_err(dev, "failed to alloc QP context, ret = %d.\n", + ret); + goto err_qpc; + } + + ret = udma_qp_store(udma_dev, qp); + if (ret) { + dev_err(dev, "failed to store QP, ret = %d.\n", ret); + goto err_store; + } + + if (udata && udata->uctx) { + resp.cap_flags = qp->en_flags; + resp.qpn = qp->qpn; + resp.priority = qp->priority; + resp.path_mtu = udma_dev->caps.max_mtu; + ret = copy_to_user((void *)udata->udrv_data->out_addr, &resp, + min(udata->udrv_data->out_len, + (uint32_t)sizeof(resp))); + if (ret) { + dev_err(dev, "copy qp resp failed!\n"); + goto err_copy; + } + } + + refcount_set(&qp->refcount, 1); + init_completion(&qp->free); + + return 0; + +err_copy: + udma_qp_remove(udma_dev, qp); +err_store: + free_qpc(udma_dev, qp); +err_qpc: + free_qpn(qp); +err_qpn: + return ret; +} + +void udma_destroy_qp_common(struct udma_dev *udma_dev, struct udma_qp *qp) +{ + udma_qp_remove(udma_dev, qp); + + if (refcount_dec_and_test(&qp->refcount)) + complete(&qp->free); + wait_for_completion(&qp->free); + + free_qpc(udma_dev, qp); + free_qpn(qp); +} + int udma_init_qp_table(struct udma_dev *dev) { struct udma_qp_table *qp_table = &dev->qp_table; diff --git a/drivers/ub/hw/hns3/hns3_udma_qp.h b/drivers/ub/hw/hns3/hns3_udma_qp.h index 479163b7757d..ba4e604e5879 100644 --- a/drivers/ub/hw/hns3/hns3_udma_qp.h +++ b/drivers/ub/hw/hns3/hns3_udma_qp.h @@ -18,6 +18,18 @@ #include "hns3_udma_device.h" +struct udma_qp_cap { + uint32_t max_send_wr; + uint32_t max_recv_wr; + uint32_t max_send_sge; + uint32_t max_recv_sge; + uint32_t max_inline_data; + uint8_t retry_cnt; + uint8_t rnr_retry; + uint8_t min_rnr_timer; + uint8_t ack_timeout; +}; + struct udma_qpn_bitmap { uint32_t qpn_prefix; uint32_t jid; @@ -27,17 +39,76 @@ struct udma_qpn_bitmap { atomic_t ref_num; }; +struct udma_qp_attr { + bool is_jetty; + bool is_tgt; + struct ubcore_ucontext *uctx; + struct udma_jfc *send_jfc; + struct udma_jfc *recv_jfc; + struct udma_jfr *jfr; + struct udma_qp_cap cap; + enum udma_qp_type qp_type; + uint32_t pdn; + struct udma_qpn_bitmap *qpn_map; + void *reorder_cq_page; + int reorder_cq_size; + dma_addr_t reorder_cq_addr; + union ubcore_eid remote_eid; + union ubcore_eid local_eid; + uint8_t priority; +}; + +struct udma_wq { + uint32_t wqe_cnt; /* WQE num */ + uint32_t max_gs; + int offset; + int wqe_offset; + int wqe_shift; /* WQE size */ + uint32_t head; +}; + +struct udma_qp_sge { + uint32_t sge_cnt; /* SGE num */ + int offset; + int sge_shift; /* SGE size */ + int wqe_offset; +}; + struct udma_qp { struct udma_dev *udma_device; + enum udma_qp_type qp_type; + struct udma_qp_attr qp_attr; + struct udma_wq rq; + struct udma_jfc *send_jfc; + struct udma_jfc *recv_jfc; + uint64_t en_flags; struct udma_mtr mtr; + enum udma_qp_state state; void (*event)(struct udma_qp *qp, enum udma_event event_type); uint64_t qpn; refcount_t refcount; struct completion free; + struct udma_qp_sge sge; + uint32_t max_inline_data; + struct list_head node; /* all qps are on a list */ + struct list_head rq_node; /* all recv qps are on a list */ + struct list_head sq_node; /* all send qps are on a list */ + uint8_t retry_cnt; + uint8_t rnr_retry; + uint8_t ack_timeout; + uint8_t min_rnr_timer; + uint8_t priority; }; +#define gen_qpn(high, mid, low) ((high) | (mid) | (low)) + +int udma_fill_qp_attr(struct udma_dev *udma_dev, struct udma_qp_attr *qp_attr, + const struct ubcore_tp_cfg *cfg, struct ubcore_udata *udata); +int udma_create_qp_common(struct udma_dev *udma_dev, struct udma_qp *qp, + struct ubcore_udata *udata); +void udma_destroy_qp_common(struct udma_dev *udma_dev, struct udma_qp *qp); void init_jetty_x_qpn_bitmap(struct udma_dev *dev, struct udma_qpn_bitmap *qpn_map, uint32_t jetty_x_shift, uint32_t prefix, diff --git a/drivers/ub/hw/hns3/hns3_udma_tp.c b/drivers/ub/hw/hns3/hns3_udma_tp.c new file mode 100644 index 000000000000..ea765a6bd7e8 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_tp.c @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include +#include +#include "hns3_udma_jfr.h" +#include "hns3_udma_tp.h" + +static void udma_set_tp(struct ubcore_device *dev, const struct ubcore_tp_cfg *cfg, + struct udma_tp *tp) +{ + tp->ubcore_tp.tpn = tp->qp.qpn; + tp->ubcore_tp.ub_dev = dev; + tp->ubcore_tp.flag.bs.target = cfg->flag.bs.target; + tp->ubcore_tp.flag.bs.oor_en = cfg->flag.bs.oor_en; + tp->ubcore_tp.flag.bs.sr_en = cfg->flag.bs.sr_en; + tp->ubcore_tp.flag.bs.spray_en = cfg->flag.bs.spray_en; + tp->ubcore_tp.local_net_addr = cfg->local_net_addr; + tp->ubcore_tp.peer_net_addr = cfg->peer_net_addr; + tp->ubcore_tp.local_eid = cfg->local_eid; + tp->ubcore_tp.peer_eid = cfg->peer_eid; + tp->ubcore_tp.trans_mode = cfg->trans_mode; + tp->ubcore_tp.rx_psn = cfg->rx_psn; + tp->ubcore_tp.mtu = cfg->mtu; + tp->ubcore_tp.data_udp_start = cfg->data_udp_start; + tp->ubcore_tp.ack_udp_start = cfg->ack_udp_start; + tp->ubcore_tp.udp_range = cfg->udp_range; + tp->ubcore_tp.retry_num = cfg->retry_num; + tp->ubcore_tp.ack_timeout = cfg->ack_timeout; + tp->ubcore_tp.tc = cfg->tc; + tp->ubcore_tp.state = UBCORE_TP_STATE_RESET; +} + +static int udma_store_tp(struct udma_dev *udma_device, struct udma_tp *tp, + struct ubcore_tp **fail_ret_tp) +{ + struct udma_qp_attr *qp_attr; + struct udma_jfr *jfr; + int ret = 0; + + qp_attr = &tp->qp.qp_attr; + jfr = qp_attr->jfr; + if (jfr) { + ret = xa_err(xa_store(&jfr->tp_table_xa, + tp->ubcore_tp.tpn, tp, + GFP_KERNEL)); + if (ret) { + dev_err(udma_device->dev, + "failed store jfr tp, ret = %d\n", ret); + return ret; + } + } + + return ret; +} + +struct ubcore_tp *udma_create_tp(struct ubcore_device *dev, const struct ubcore_tp_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + struct ubcore_tp *fail_ret_tp = NULL; + struct udma_tp *tp; + int ret; + + tp = kzalloc(sizeof(*tp), GFP_KERNEL); + if (!tp) + return ERR_PTR(-ENOMEM); + + ret = udma_fill_qp_attr(udma_dev, &tp->qp.qp_attr, cfg, udata); + if (ret) { + dev_err(udma_dev->dev, "failed to fill qp attr.\n"); + goto failed_alloc_tp; + } + + ret = udma_create_qp_common(udma_dev, &tp->qp, udata); + if (ret) { + dev_err(udma_dev->dev, + "Failed to create qp common with ret is %d.\n", ret); + goto failed_alloc_tp; + } + + udma_set_tp(dev, cfg, tp); + + ret = udma_store_tp(udma_dev, tp, &fail_ret_tp); + if (ret || fail_ret_tp) + goto failed_create_qp; + + return &tp->ubcore_tp; + +failed_create_qp: + udma_destroy_qp_common(udma_dev, &tp->qp); +failed_alloc_tp: + kfree(tp); + + return fail_ret_tp; +} diff --git a/drivers/ub/hw/hns3/hns3_udma_tp.h b/drivers/ub/hw/hns3/hns3_udma_tp.h new file mode 100644 index 000000000000..7355447a3736 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_tp.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_TP_H +#define _UDMA_TP_H + +#include +#include "hns3_udma_qp.h" + +struct udma_tp { + struct ubcore_tp ubcore_tp; + struct udma_qp qp; + struct ubcore_jetty_id tjetty_id; +}; +struct ubcore_tp *udma_create_tp(struct ubcore_device *dev, + const struct ubcore_tp_cfg *cfg, + struct ubcore_udata *udata); +int udma_destroy_tp(struct ubcore_tp *tp); + +#endif /* _UDMA_TP_H */ -- Gitee From 364d4bf5561f879459b0d145034bf7744a00c90e Mon Sep 17 00:00:00 2001 From: Weibo Zhao Date: Mon, 28 Aug 2023 16:53:01 +0800 Subject: [PATCH 26/42] hns3 udma: add function clear when ko rmmod driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA --------------------------------------------- When rmmod ko and hw exit, function clear should be done. Driver post a function clear command to hardware to clean resource in function. Signed-off-by: Weibo Zhao --- drivers/ub/hw/hns3/hns3_udma_hw.c | 70 +++++++++++++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_hw.h | 11 +++++ 2 files changed, 81 insertions(+) diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.c b/drivers/ub/hw/hns3/hns3_udma_hw.c index 6e82a67998d7..3fb4b056a650 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hw.c +++ b/drivers/ub/hw/hns3/hns3_udma_hw.c @@ -1110,6 +1110,74 @@ static int udma_hw_set_eid(struct udma_dev *udma_dev, union ubcore_eid eid) return ret; } +static void __udma_function_clear(struct udma_dev *udma_dev, int vf_id) +{ + struct udma_func_clear *resp; + struct udma_cmq_desc desc; + uint64_t end; + int ret = 0; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_FUNC_CLEAR, false); + resp = (struct udma_func_clear *)desc.data; + resp->rst_funcid_en = cpu_to_le32(vf_id); + + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) { + dev_err(udma_dev->dev, "Func clear write failed, ret = %d.\n", + ret); + return; + } + + msleep(UDMA_READ_FUNC_CLEAR_FLAG_INTERVAL); + end = UDMA_FUNC_CLEAR_TIMEOUT_MSECS; + while (end) { + msleep(UDMA_READ_FUNC_CLEAR_FLAG_FAIL_WAIT); + end -= UDMA_READ_FUNC_CLEAR_FLAG_FAIL_WAIT; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_FUNC_CLEAR, + true); + + resp->rst_funcid_en = cpu_to_le32(vf_id); + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) + continue; + + if (udma_get_bit(resp->func_done, FUNC_CLEAR_RST_FUN_DONE_S)) { + if (vf_id == 0) + udma_dev->is_reset = true; + return; + } + } +} + +static void udma_free_vf_resource(struct udma_dev *udma_dev, int vf_id) +{ + enum udma_opcode_type opcode = UDMA_OPC_ALLOC_VF_RES; + struct udma_cmq_desc desc[2]; + struct udma_cmq_req *req_a; + + req_a = (struct udma_cmq_req *)desc[0].data; + udma_cmq_setup_basic_desc(&desc[0], opcode, false); + desc[0].flag |= cpu_to_le16(UDMA_CMD_FLAG_NEXT); + udma_cmq_setup_basic_desc(&desc[1], opcode, false); + udma_reg_write(req_a, FUNC_RES_A_VF_ID, vf_id); + udma_cmq_send(udma_dev, desc, 2); +} + +static void udma_function_clear(struct udma_dev *udma_dev) +{ + int i; + + if (udma_dev->cmd.state == UDMA_CMDQ_STATE_FATAL_ERR) + return; + + for (i = udma_dev->func_num - 1; i >= 0; i--) { + __udma_function_clear(udma_dev, i); + if (i != 0) + udma_free_vf_resource(udma_dev, i); + } +} + static void config_llm_table(struct udma_buf *data_buf, void *cfg_buf) { uint64_t *entry = (uint64_t *)cfg_buf; @@ -1359,6 +1427,8 @@ static int udma_hw_init(struct udma_dev *udma_dev) static void udma_hw_exit(struct udma_dev *udma_dev) { + udma_function_clear(udma_dev); + udma_free_link_table(udma_dev); put_hem_table(udma_dev); diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.h b/drivers/ub/hw/hns3/hns3_udma_hw.h index e6beb8142205..18f8a284cd21 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hw.h +++ b/drivers/ub/hw/hns3/hns3_udma_hw.h @@ -29,6 +29,11 @@ struct udma_query_fw_info { uint32_t rsv[5]; }; +struct udma_func_clear { + uint32_t rst_funcid_en; + uint32_t func_done; + uint32_t rsv[4]; +}; struct udma_pf_func_info { uint32_t own_func_num; @@ -105,6 +110,11 @@ struct udma_query_oor_cmq { #define CFG_GLOBAL_PARAM_1US_CYCLES CMQ_REQ_FIELD_LOC(9, 0) #define CFG_GLOBAL_PARAM_UDP_PORT CMQ_REQ_FIELD_LOC(31, 16) +#define FUNC_CLEAR_RST_FUN_DONE_S 0 +#define UDMA_FUNC_CLEAR_TIMEOUT_MSECS (249 * 2 * 100) +#define UDMA_READ_FUNC_CLEAR_FLAG_INTERVAL 40 +#define UDMA_READ_FUNC_CLEAR_FLAG_FAIL_WAIT 20 + #define UDMA_FUNC_IRQ_RSV 2 #define UDMA_1US_CFG 999 #define UDMA_EXT_LLM_ENTRY_SZ 8 @@ -204,6 +214,7 @@ enum udma_opcode_type { UDMA_OPC_POST_MB = 0x8504, UDMA_OPC_QUERY_MB_ST = 0x8505, UDMA_OPC_CFG_BT_ATTR = 0x8506, + UDMA_OPC_FUNC_CLEAR = 0x8508, UDMA_OPC_CLEAR_EXTDB_LIST_INFO = 0x850d, UDMA_OPC_QUERY_VF_RES = 0x850e, UDMA_OPC_CFG_GMV_BT = 0x8510, -- Gitee From 8c3f8817da6adaf6164be66b768e367ebc147ee9 Mon Sep 17 00:00:00 2001 From: Weibo Zhao Date: Tue, 29 Aug 2023 16:58:06 +0800 Subject: [PATCH 27/42] hns3 udma: add support of create / destroy JFS driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA ---------------------------------------------------------- This patch add support of create / destroy JFS(jetty for send). RM(Reliable Mode) mode of JFS can send packet to any JFR. UM(Unreliable Mode) not support yet. Signed-off-by: Weibo Zhao --- drivers/ub/hw/hns3/Makefile | 2 +- drivers/ub/hw/hns3/hns3_udma_abi.h | 9 ++ drivers/ub/hw/hns3/hns3_udma_jfs.c | 166 ++++++++++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_jfs.h | 41 +++++++ drivers/ub/hw/hns3/hns3_udma_main.c | 3 + 5 files changed, 220 insertions(+), 1 deletion(-) create mode 100644 drivers/ub/hw/hns3/hns3_udma_jfs.c create mode 100644 drivers/ub/hw/hns3/hns3_udma_jfs.h diff --git a/drivers/ub/hw/hns3/Makefile b/drivers/ub/hw/hns3/Makefile index 4250bbc0ec93..38ab0ff041d5 100644 --- a/drivers/ub/hw/hns3/Makefile +++ b/drivers/ub/hw/hns3/Makefile @@ -10,6 +10,6 @@ ccflags-y += -I$(srctree)/drivers/net/ethernet/hisilicon/hns3/ \ $(MODULE_NAME)-objs := hns3_udma_hw.o hns3_udma_main.o hns3_udma_cmd.o \ hns3_udma_hem.o hns3_udma_qp.o hns3_udma_eq.o \ hns3_udma_db.o hns3_udma_jfc.o hns3_udma_jfr.o \ - hns3_udma_segment.o hns3_udma_tp.o + hns3_udma_segment.o hns3_udma_tp.o hns3_udma_jfs.o obj-$(CONFIG_UB_UDMA_HNS3) := hns3_udma.o diff --git a/drivers/ub/hw/hns3/hns3_udma_abi.h b/drivers/ub/hw/hns3/hns3_udma_abi.h index ee817b23c0de..1c6b182302ab 100644 --- a/drivers/ub/hw/hns3/hns3_udma_abi.h +++ b/drivers/ub/hw/hns3/hns3_udma_abi.h @@ -23,6 +23,7 @@ #define MAP_INDEX_SHIFT 8 #define UDMA_DWQE_PAGE_SIZE 65536 #define UDMA_JETTY_X_PREFIX_BIT_NUM 2 +#define UDMA_JFS_QPN_PREFIX 0x0 #define UDMA_JFR_QPN_PREFIX 0x1 #define UDMA_ADDR_4K_MASK 0xfffUL #define URMA_SEG_ACCESS_GUARD (1UL << 5) @@ -105,6 +106,14 @@ struct udma_create_tp_resp { uint8_t priority; }; +struct udma_create_jfs_ucmd { + struct udma_create_tp_ucmd create_tp_ucmd; +}; + +struct udma_create_jfs_resp { + struct udma_create_tp_resp create_tp_resp; +}; + struct udma_create_ctx_resp { uint32_t num_comp_vectors; uint32_t num_qps_shift; diff --git a/drivers/ub/hw/hns3/hns3_udma_jfs.c b/drivers/ub/hw/hns3/hns3_udma_jfs.c new file mode 100644 index 000000000000..fefd64ca086b --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_jfs.c @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include "urma/ubcore_uapi.h" +#include "hns3_udma_abi.h" +#include "hns3_udma_hem.h" +#include "hns3_udma_tp.h" +#include "hns3_udma_jfs.h" + +static int init_jfs_cfg(struct udma_dev *dev, struct udma_jfs *jfs, + const struct ubcore_jfs_cfg *cfg) +{ + if (!cfg->max_sge || + cfg->depth > dev->caps.max_wqes || + cfg->max_sge > dev->caps.max_sq_sg) { + dev_err(dev->dev, "invalid jfs cfg, depth = %u, sge = %u.\n", + cfg->depth, cfg->max_sge); + return -EINVAL; + } + memcpy(&jfs->ubcore_jfs.jfs_cfg, cfg, sizeof(struct ubcore_jfs_cfg)); + jfs->tp_mode = cfg->trans_mode; + + return 0; +} + +int destroy_jfs_qp(struct udma_dev *dev, struct udma_jfs *jfs) +{ + int ret = 0; + + if (jfs->tp_mode == UBCORE_TP_UM) + dev_err(dev->dev, "Not support UM mode.\n"); + + return ret; +} + +static int alloc_jfs_buf(struct udma_dev *udma_dev, struct udma_jfs *jfs, + const struct ubcore_jfs_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_create_jfs_ucmd ucmd = {}; + int ret = 0; + + if (udata) { + ret = copy_from_user(&ucmd, (void *)udata->udrv_data->in_addr, + min(udata->udrv_data->in_len, + (uint32_t)sizeof(ucmd))); + if (ret) { + dev_err(udma_dev->dev, + "failed to copy jfs udata, ret = %d.\n", ret); + return -EFAULT; + } + } + + if (cfg->trans_mode == UBCORE_TP_RM) { + xa_init(&jfs->node_table); + } else if (cfg->trans_mode == UBCORE_TP_UM) { + dev_err(udma_dev->dev, "Not Support UM mode.\n"); + return -EINVAL; + } + + return ret; +} + +static int alloc_jfs_id(struct udma_dev *udma_dev, struct udma_jfs *jfs) +{ + struct udma_jfs_table *jfs_table = &udma_dev->jfs_table; + struct udma_ida *jfs_ida = &jfs_table->jfs_ida; + int ret; + int id; + + id = ida_alloc_range(&jfs_ida->ida, jfs_ida->min, jfs_ida->max, + GFP_KERNEL); + if (id < 0) { + dev_err(udma_dev->dev, "failed to alloc jfs_id(%d).\n", id); + return id; + } + jfs->jfs_id = (uint32_t)id; + jfs->ubcore_jfs.id = jfs->jfs_id; + + ret = xa_err(xa_store(&jfs_table->xa, jfs->jfs_id, jfs, GFP_KERNEL)); + if (ret) { + dev_err(udma_dev->dev, "failed to store JFS, ret = %d.\n", ret); + ida_free(&jfs_ida->ida, id); + } + + return ret; +} + +static void free_jfs_id(struct udma_dev *udma_dev, struct udma_jfs *jfs) +{ + struct udma_jfs_table *jfs_table = &udma_dev->jfs_table; + struct udma_ida *jfs_ida = &jfs_table->jfs_ida; + + xa_erase(&jfs_table->xa, jfs->jfs_id); + ida_free(&jfs_ida->ida, (int)jfs->jfs_id); +} + +struct ubcore_jfs *udma_create_jfs(struct ubcore_device *dev, const struct ubcore_jfs_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + struct udma_jfs *jfs; + int ret; + + jfs = kcalloc(1, sizeof(struct udma_jfs), GFP_KERNEL); + if (!jfs) + return NULL; + + ret = init_jfs_cfg(udma_dev, jfs, cfg); + if (ret) + goto err_init_cfg; + + ret = alloc_jfs_id(udma_dev, jfs); + if (ret) + goto err_alloc_jfs_id; + + init_jetty_x_qpn_bitmap(udma_dev, &jfs->qpn_map, + udma_dev->caps.num_jfs_shift, + UDMA_JFS_QPN_PREFIX, jfs->jfs_id); + + ret = alloc_jfs_buf(udma_dev, jfs, cfg, udata); + if (ret) { + dev_err(udma_dev->dev, "alloc jfs buf failed.\n"); + goto err_alloc_jfs_buf; + } + + return &jfs->ubcore_jfs; + +err_alloc_jfs_buf: + clean_jetty_x_qpn_bitmap(&jfs->qpn_map); + free_jfs_id(udma_dev, jfs); +err_alloc_jfs_id: +err_init_cfg: + kfree(jfs); + + return NULL; +} + +int udma_destroy_jfs(struct ubcore_jfs *jfs) +{ + struct udma_jfs *udma_jfs; + struct udma_dev *udma_dev; + int ret; + + udma_jfs = to_udma_jfs(jfs); + udma_dev = to_udma_dev(jfs->ub_dev); + + ret = destroy_jfs_qp(udma_dev, udma_jfs); + clean_jetty_x_qpn_bitmap(&udma_jfs->qpn_map); + free_jfs_id(udma_dev, udma_jfs); + kfree(udma_jfs); + + return ret; +} diff --git a/drivers/ub/hw/hns3/hns3_udma_jfs.h b/drivers/ub/hw/hns3/hns3_udma_jfs.h new file mode 100644 index 000000000000..eee4058bb0b3 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_jfs.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_JFS_H +#define _UDMA_JFS_H + +#include "hns3_udma_qp.h" +struct udma_jfs { + struct ubcore_jfs ubcore_jfs; + uint32_t jfs_id; + enum ubcore_transport_mode tp_mode; + union { + struct xarray node_table; + struct udma_qp um_qp; + }; + struct udma_qpn_bitmap qpn_map; +}; + +static inline struct udma_jfs *to_udma_jfs(struct ubcore_jfs *jfs) +{ + return container_of(jfs, struct udma_jfs, ubcore_jfs); +} + +struct ubcore_jfs *udma_create_jfs(struct ubcore_device *dev, + const struct ubcore_jfs_cfg *cfg, + struct ubcore_udata *udata); +int udma_destroy_jfs(struct ubcore_jfs *ubcore_jfs); + +#endif /* _UDMA_JFS_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c index 3216ef0d3f9e..2109f82ef437 100644 --- a/drivers/ub/hw/hns3/hns3_udma_main.c +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -22,6 +22,7 @@ #include "hns3_udma_hem.h" #include "hns3_udma_jfr.h" #include "hns3_udma_jfc.h" +#include "hns3_udma_jfs.h" #include "hns3_udma_segment.h" #include "hns3_udma_cmd.h" static int udma_set_eid(struct ubcore_device *dev, union ubcore_eid eid) @@ -320,6 +321,8 @@ static struct ubcore_ops g_udma_dev_ops = { .create_jfc = udma_create_jfc, .modify_jfc = udma_modify_jfc, .destroy_jfc = udma_destroy_jfc, + .create_jfs = udma_create_jfs, + .destroy_jfs = udma_destroy_jfs, .create_jfr = udma_create_jfr, .modify_jfr = udma_modify_jfr, .destroy_jfr = udma_destroy_jfr, -- Gitee From 460bb8aebf6eae2df3939c7dcc5668add60a0c75 Mon Sep 17 00:00:00 2001 From: Weibo Zhao Date: Wed, 30 Aug 2023 14:15:44 +0800 Subject: [PATCH 28/42] hns3 udma: add support of UM JFS driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA --------------------------------------------------- UM(Unreliable Message) mode of JFS can send message to any JFR but not reliable. Signed-off-by: Weibo Zhao --- drivers/ub/hw/hns3/hns3_udma_abi.h | 7 + drivers/ub/hw/hns3/hns3_udma_device.h | 27 ++ drivers/ub/hw/hns3/hns3_udma_jfs.c | 93 ++++- drivers/ub/hw/hns3/hns3_udma_qp.c | 568 ++++++++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_qp.h | 164 ++++++++ drivers/ub/hw/hns3/hns3_udma_tp.h | 2 + 6 files changed, 857 insertions(+), 4 deletions(-) diff --git a/drivers/ub/hw/hns3/hns3_udma_abi.h b/drivers/ub/hw/hns3/hns3_udma_abi.h index 1c6b182302ab..15d60481e37c 100644 --- a/drivers/ub/hw/hns3/hns3_udma_abi.h +++ b/drivers/ub/hw/hns3/hns3_udma_abi.h @@ -99,6 +99,13 @@ struct udma_create_tp_ucmd { uint64_t sdb_addr; }; +enum udma_qp_cap_flags { + UDMA_QP_CAP_RQ_RECORD_DB = 1 << 0, + UDMA_QP_CAP_SQ_RECORD_DB = 1 << 1, + UDMA_QP_CAP_OWNER_DB = 1 << 2, + UDMA_QP_CAP_DIRECT_WQE = 1 << 5, +}; + struct udma_create_tp_resp { uint64_t cap_flags; uint32_t qpn; diff --git a/drivers/ub/hw/hns3/hns3_udma_device.h b/drivers/ub/hw/hns3/hns3_udma_device.h index 3ba30971133f..295cc5125e15 100644 --- a/drivers/ub/hw/hns3/hns3_udma_device.h +++ b/drivers/ub/hw/hns3/hns3_udma_device.h @@ -173,6 +173,13 @@ enum udma_event { UDMA_EVENT_TYPE_MB = 0x13, }; +enum udma_mtu { + UDMA_MTU_256 = 1, + UDMA_MTU_512 = 2, + UDMA_MTU_1024 = 3, + UDMA_MTU_2048 = 4, + UDMA_MTU_4096 = 5 +}; enum { /* discard BIT(2), reserved for compatibility */ @@ -211,6 +218,10 @@ enum udma_cong_sel { UDMA_CONG_SEL_DIP = 1 << UDMA_CONG_TYPE_DIP, }; +enum udma_sig_type { + SIGNAL_REQ_WR = 1, +}; + enum udma_qp_type { QPT_RC, QPT_UD = 0x3, @@ -822,11 +833,27 @@ static inline uint32_t to_udma_hem_entries_size(uint32_t count, { return udma_hw_page_align(count << buf_shift); } + static inline uint32_t to_udma_hw_page_shift(uint32_t page_shift) { return page_shift - UDMA_HW_PAGE_SHIFT; } +static inline uint32_t to_udma_hem_entries_count(uint32_t count, + uint32_t buf_shift) +{ + return udma_hw_page_align(count << buf_shift) >> buf_shift; +} + +static inline uint32_t to_udma_hem_entries_shift(uint32_t count, + uint32_t buf_shift) +{ + if (!count) + return 0; + + return ilog2(to_udma_hem_entries_count(count, buf_shift)); +} + static inline uint64_t to_udma_hw_page_addr(uint64_t addr) { return addr >> UDMA_HW_PAGE_SHIFT; diff --git a/drivers/ub/hw/hns3/hns3_udma_jfs.c b/drivers/ub/hw/hns3/hns3_udma_jfs.c index fefd64ca086b..8ec30057fa7c 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jfs.c +++ b/drivers/ub/hw/hns3/hns3_udma_jfs.c @@ -17,6 +17,7 @@ #include "hns3_udma_abi.h" #include "hns3_udma_hem.h" #include "hns3_udma_tp.h" +#include "hns3_udma_jfc.h" #include "hns3_udma_jfs.h" static int init_jfs_cfg(struct udma_dev *dev, struct udma_jfs *jfs, @@ -35,12 +36,90 @@ static int init_jfs_cfg(struct udma_dev *dev, struct udma_jfs *jfs, return 0; } +static int udma_modify_jfs_um_qp(struct udma_dev *dev, struct udma_jfs *jfs, + enum udma_qp_state target_state) +{ + struct udma_modify_tp_attr m_attr = {}; + struct udma_qp *qp; + int ret; + + qp = &jfs->um_qp; + qp->udma_device = dev; + qp->send_jfc = qp->qp_attr.send_jfc; + qp->recv_jfc = qp->qp_attr.recv_jfc; + qp->ubcore_path_mtu = UBCORE_MTU_4096; + qp->path_mtu = UDMA_MTU_4096; + + m_attr.path_mtu = UBCORE_MTU_4096; + m_attr.hop_limit = MAX_HOP_LIMIT; + ret = udma_modify_qp_common(qp, &m_attr, qp->state, target_state); + if (ret) + dev_err(dev->dev, "failed to modify qpc to RTS.\n"); + + jfs->um_qp.state = target_state; + + return ret; +} + +static void udma_fill_jfs_um_qp_attr(struct udma_dev *dev, struct udma_jfs *jfs, + struct udma_qp_attr *qp_attr, + struct ubcore_ucontext *uctx, + const struct ubcore_jfs_cfg *cfg) +{ + struct udma_ucontext *udma_ctx = to_udma_ucontext(uctx); + + qp_attr->is_jetty = false; + qp_attr->jfs = jfs; + qp_attr->uctx = uctx; + qp_attr->pdn = udma_ctx->pdn; + qp_attr->cap.max_send_wr = cfg->depth; + qp_attr->cap.max_send_sge = cfg->max_sge; + qp_attr->cap.max_inline_data = cfg->max_inline_data; + qp_attr->cap.retry_cnt = cfg->retry_cnt; + qp_attr->cap.rnr_retry = cfg->rnr_retry; + qp_attr->cap.ack_timeout = cfg->err_timeout; + qp_attr->qp_type = QPT_UD; + qp_attr->recv_jfc = NULL; + qp_attr->send_jfc = to_udma_jfc(cfg->jfc); + if (jfs->ubcore_jfs.jfs_cfg.priority >= dev->caps.sl_num) { + qp_attr->priority = dev->caps.sl_num > 0 ? + dev->caps.sl_num - 1 : 0; + dev_err(dev->dev, + "The setted priority (%d) should smaller than the max priority (%d), priority (%d) is used\n", + jfs->ubcore_jfs.jfs_cfg.priority, dev->caps.sl_num, + qp_attr->priority); + } else { + qp_attr->priority = jfs->ubcore_jfs.jfs_cfg.priority; + } +} + +static int create_jfs_um_qp(struct udma_dev *dev, struct udma_jfs *jfs, + const struct ubcore_jfs_cfg *cfg, struct ubcore_udata *udata) +{ + int ret; + + udma_fill_jfs_um_qp_attr(dev, jfs, &jfs->um_qp.qp_attr, udata->uctx, cfg); + jfs->um_qp.qp_attr.qpn_map = &jfs->qpn_map; + ret = udma_create_qp_common(dev, &jfs->um_qp, udata); + if (ret) + dev_err(dev->dev, "failed to create qp for um jfs.\n"); + + return ret; +} + int destroy_jfs_qp(struct udma_dev *dev, struct udma_jfs *jfs) { int ret = 0; - if (jfs->tp_mode == UBCORE_TP_UM) - dev_err(dev->dev, "Not support UM mode.\n"); + if (jfs->tp_mode == UBCORE_TP_UM) { + ret = udma_modify_jfs_um_qp(dev, jfs, QPS_RESET); + if (ret) + dev_err(dev->dev, + "failed to modify qp(0x%llx) to RESET for um jfs.\n", + jfs->um_qp.qpn); + + udma_destroy_qp_common(dev, &jfs->um_qp); + } return ret; } @@ -66,8 +145,14 @@ static int alloc_jfs_buf(struct udma_dev *udma_dev, struct udma_jfs *jfs, if (cfg->trans_mode == UBCORE_TP_RM) { xa_init(&jfs->node_table); } else if (cfg->trans_mode == UBCORE_TP_UM) { - dev_err(udma_dev->dev, "Not Support UM mode.\n"); - return -EINVAL; + ret = create_jfs_um_qp(udma_dev, jfs, cfg, udata); + if (ret) + return ret; + + jfs->um_qp.state = QPS_RESET; + ret = udma_modify_jfs_um_qp(udma_dev, jfs, QPS_RTS); + if (ret) + udma_destroy_qp_common(udma_dev, &jfs->um_qp); } return ret; diff --git a/drivers/ub/hw/hns3/hns3_udma_qp.c b/drivers/ub/hw/hns3/hns3_udma_qp.c index 407bddab2db6..427ea1c112f4 100644 --- a/drivers/ub/hw/hns3/hns3_udma_qp.c +++ b/drivers/ub/hw/hns3/hns3_udma_qp.c @@ -19,8 +19,208 @@ #include "hns3_udma_jfr.h" #include "hns3_udma_jfc.h" #include "hns3_udma_hem.h" +#include "hns3_udma_cmd.h" #include "hns3_udma_qp.h" +static void set_qpc_wqe_cnt(struct udma_qp *qp, + struct udma_qp_context *context, + struct udma_qp_context *context_mask) +{ + udma_reg_write(context, QPC_SGE_SHIFT, + to_udma_hem_entries_shift(qp->sge.sge_cnt, + qp->sge.sge_shift)); + udma_reg_clear(context_mask, QPC_SGE_SHIFT); + + udma_reg_write(context, QPC_SQ_SHIFT, ilog2(qp->sq.wqe_cnt)); + udma_reg_clear(context_mask, QPC_SQ_SHIFT); +} + +static void config_qp_sq_buf_mask(struct udma_qp_context *context_mask) +{ + udma_reg_clear(context_mask, QPC_SQ_CUR_BLK_ADDR_L); + udma_reg_clear(context_mask, QPC_SQ_CUR_BLK_ADDR_H); + udma_reg_clear(context_mask, QPC_SQ_CUR_SGE_BLK_ADDR_L); + udma_reg_clear(context_mask, QPC_SQ_CUR_SGE_BLK_ADDR_H); + udma_reg_clear(context_mask, QPC_RX_SQ_CUR_BLK_ADDR_L); + udma_reg_clear(context_mask, QPC_RX_SQ_CUR_BLK_ADDR_H); + udma_reg_clear(context_mask, QPC_WQE_SGE_BA_L); + udma_reg_clear(context_mask, QPC_WQE_SGE_BA_H); + udma_reg_clear(context_mask, QPC_SQ_HOP_NUM); + udma_reg_clear(context_mask, QPC_SGE_HOP_NUM); + udma_reg_clear(context_mask, QPC_WQE_SGE_BA_PG_SZ); + udma_reg_clear(context_mask, QPC_WQE_SGE_BUF_PG_SZ); +} + +static int config_qp_sq_buf(struct udma_dev *udma_device, + struct udma_qp *qp, + struct udma_qp_context *context, + struct udma_qp_context *context_mask) +{ + uint64_t sge_cur_blk = 0; + uint64_t sq_cur_blk = 0; + uint64_t wqe_sge_ba = 0; + int count; + + /* search qp buf's mtts */ + count = udma_mtr_find(udma_device, &qp->mtr, qp->sq.wqe_offset, + &sq_cur_blk, 1, &wqe_sge_ba); + if (count < 1) { + dev_err(udma_device->dev, "failed to find QP(0x%llx) SQ buf.\n", + qp->qpn); + return -EINVAL; + } + + context->wqe_sge_ba = cpu_to_le32(wqe_sge_ba >> WQE_SGE_BA_OFFSET); + + if (qp->sge.sge_cnt > 0) { + count = udma_mtr_find(udma_device, &qp->mtr, + qp->sge.wqe_offset, &sge_cur_blk, + 1, NULL); + if (count < 1) { + dev_err(udma_device->dev, + "failed to find QP(0x%llx) SGE buf.\n", qp->qpn); + return -EINVAL; + } + } + + /* + * In v2 engine, software pass context and context mask to hardware + * when modifying qp. If software need modify some fields in context, + * we should set all bits of the relevant fields in context mask to + * 0 at the same time, else set them to 0x1. + */ + udma_reg_write(context, QPC_SQ_CUR_BLK_ADDR_L, + lower_32_bits(to_udma_hw_page_addr(sq_cur_blk))); + udma_reg_write(context, QPC_SQ_CUR_BLK_ADDR_H, + upper_32_bits(to_udma_hw_page_addr(sq_cur_blk))); + udma_reg_write(context, QPC_SQ_CUR_SGE_BLK_ADDR_L, + lower_32_bits(to_udma_hw_page_addr(sge_cur_blk))); + udma_reg_write(context, QPC_SQ_CUR_SGE_BLK_ADDR_H, + upper_32_bits(to_udma_hw_page_addr(sge_cur_blk))); + udma_reg_write(context, QPC_RX_SQ_CUR_BLK_ADDR_L, + lower_32_bits(to_udma_hw_page_addr(sq_cur_blk))); + udma_reg_write(context, QPC_RX_SQ_CUR_BLK_ADDR_H, + upper_32_bits(to_udma_hw_page_addr(sq_cur_blk))); + udma_reg_write(context, QPC_WQE_SGE_BA_H, wqe_sge_ba >> + (WQE_SGE_BA_OFFSET + H_ADDR_OFFSET)); + udma_reg_write(context, QPC_SQ_HOP_NUM, + to_udma_hem_hopnum(udma_device->caps.wqe_sq_hop_num, + qp->sq.wqe_cnt)); + udma_reg_write(context, QPC_SGE_HOP_NUM, + to_udma_hem_hopnum(udma_device->caps.wqe_sge_hop_num, + qp->sge.sge_cnt)); + udma_reg_write(context, QPC_WQE_SGE_BA_PG_SZ, + to_udma_hw_page_shift(qp->mtr.hem_cfg.ba_pg_shift)); + udma_reg_write(context, QPC_WQE_SGE_BUF_PG_SZ, + to_udma_hw_page_shift(qp->mtr.hem_cfg.buf_pg_shift)); + + config_qp_sq_buf_mask(context_mask); + + return 0; +} + +static void udma_set_path(const struct udma_modify_tp_attr *attr, + struct udma_qp_context *context, + struct udma_qp_context *context_mask) +{ + if (attr == NULL) + return; + + udma_reg_write(context, QPC_HOPLIMIT, attr->hop_limit); + udma_reg_clear(context_mask, QPC_HOPLIMIT); + + udma_reg_write(context, QPC_GMV_IDX, attr->sgid_index); + udma_reg_clear(context_mask, QPC_GMV_IDX); + + memcpy(context->dgid, attr->dgid, sizeof(attr->dgid)); + memset(context_mask->dgid, 0, sizeof(attr->dgid)); + + udma_reg_write(&(context->ext), QPCEX_DEID_H, + *(uint32_t *)(&attr->dgid[SGID_H_SHIFT])); + udma_reg_clear(&context_mask->ext, QPCEX_DEID_H); + + udma_reg_write(context, QPC_SL, attr->priority); + udma_reg_clear(context_mask, QPC_SL); +} + +static int udma_pass_qpc_to_hw(struct udma_dev *udma_device, + struct udma_qp_context *context, + struct udma_qp_context *qpc_mask, + struct udma_qp *qp) +{ + struct udma_cmd_mailbox *mailbox; + struct udma_cmq_desc desc; + int qpc_size; + int ret; + + struct udma_mbox *mb = (struct udma_mbox *)desc.data; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mailbox = udma_alloc_cmd_mailbox(udma_device); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + mbox_desc_init(mb, mailbox->dma, 0, qp->qpn, UDMA_CMD_MODIFY_QPC); + qpc_size = udma_device->caps.qpc_sz; + memcpy(mailbox->buf, context, qpc_size); + memcpy(mailbox->buf + qpc_size, qpc_mask, qpc_size); + + ret = udma_cmd_mbox(udma_device, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); + + udma_free_cmd_mailbox(udma_device, mailbox); + + return ret; +} + +static bool check_qp_timeout_cfg_range(struct udma_dev *udma_device, + const uint8_t *timeout) +{ + if (*timeout > QP_TIMEOUT_MAX) { + dev_warn(udma_device->dev, + "Local ACK timeout shall be 0 to 31.\n"); + return false; + } + + return true; +} + +static enum udma_mtu to_udma_mtu(enum ubcore_mtu core_mtu) +{ + switch (core_mtu) { + case UBCORE_MTU_256: + return UDMA_MTU_256; + case UBCORE_MTU_512: + return UDMA_MTU_512; + case UBCORE_MTU_1024: + return UDMA_MTU_1024; + case UBCORE_MTU_2048: + return UDMA_MTU_2048; + default: + return UDMA_MTU_4096; + } +} + +static inline enum ubcore_mtu get_mtu(struct udma_qp *qp, + const struct udma_modify_tp_attr *attr) +{ + if (qp->qp_type == QPT_UD) + return UBCORE_MTU_4096; + + return attr->path_mtu; +} + +static inline int udma_mtu_enum_to_int(enum ubcore_mtu mtu) +{ + switch (mtu) { + case UBCORE_MTU_256: return 256; + case UBCORE_MTU_512: return 512; + case UBCORE_MTU_1024: return 1024; + case UBCORE_MTU_2048: return 2048; + case UBCORE_MTU_4096: return 4096; + default: return -1; + } +} + static int udma_alloc_reorder_cq_buf(struct udma_dev *udma_dev, struct udma_qp_attr *qp_attr) { @@ -36,6 +236,7 @@ static int udma_alloc_reorder_cq_buf(struct udma_dev *udma_dev, dev_err(udma_dev->dev, "Dma alloc coherent failed\n"); return -ENOMEM; } + return 0; } @@ -48,6 +249,373 @@ static void udma_free_reorder_cq_buf(struct udma_dev *udma_dev, qp_attr->reorder_cq_addr); } +static void edit_qpc_for_inline(struct udma_qp_context *context, + struct udma_qp *qp) +{ + struct udma_dev *udma_dev = qp->udma_device; + + if (qp->qp_type != QPT_UD) { + udma_reg_write(context, QPC_CQEIE, + !!(udma_dev->caps.flags & + UDMA_CAP_FLAG_CQE_INLINE)); + } +} + +static void edit_qpc_for_ext(struct udma_qp_context *context, + struct udma_qp *qp, + const struct udma_modify_tp_attr *attr) +{ + struct udma_dev *udma_dev = qp->udma_device; + uint16_t dus_regval; + uint16_t aus_regval; + uint16_t real_range; + + if (udma_dev->caps.oor_en) { + udma_reg_enable(&context->ext, QPCEX_OOR_EN); + udma_reg_write(&context->ext, QPCEX_REORDER_CAP, + udma_dev->caps.reorder_cap); + udma_reg_write(&context->ext, QPCEX_ON_FLIGHT_SIZE_L, + udma_dev->caps.onflight_size); + udma_reg_write(&context->ext, QPCEX_ON_FLIGHT_SIZE_H, + udma_dev->caps.onflight_size >> + QPCEX_ON_FLIGHT_SIZE_H_SHIFT); + udma_reg_write(&context->ext, QPCEX_DYN_AT, + udma_dev->caps.dynamic_ack_timeout); + if (udma_dev->caps.flags & UDMA_CAP_FLAG_AR) { + real_range = (attr->udp_range + + UDP_SRCPORT_RANGE_BASE) & + UDP_SRCPORT_RANGE_SIZE_MASK; + dus_regval = attr->data_udp_start & + GENMASK(real_range, 0); + aus_regval = attr->ack_udp_start & + GENMASK(real_range, 0); + + udma_reg_write(&context->ext, QPCEX_AR_EN, attr->ar_en); + udma_reg_write(&context->ext, QPCEX_DATA_UDP_SRCPORT_L, + dus_regval); + udma_reg_write(&context->ext, QPCEX_DATA_UDP_SRCPORT_H, + dus_regval >> + QPCEX_DATA_UDP_SRCPORT_H_SHIFT); + udma_reg_write(&context->ext, QPCEX_ACK_UDP_SRCPORT, + aus_regval); + udma_reg_write(&context->ext, QPCEX_UDP_SRCPORT_RANGE, + attr->udp_range); + } + if (udma_dev->caps.reorder_cq_buffer_en && + qp->qp_attr.reorder_cq_addr) { + udma_reg_enable(&context->ext, QPCEX_REORDER_CQ_EN); + udma_reg_write(&context->ext, QPCEX_REORDER_CQ_ADDR_L, + lower_32_bits(qp->qp_attr.reorder_cq_addr) >> + QPCEX_REORDER_CQ_ADDR_SHIFT); + udma_reg_write(&context->ext, QPCEX_REORDER_CQ_ADDR_H, + upper_32_bits(qp->qp_attr.reorder_cq_addr)); + udma_reg_write(&context->ext, QPCEX_REORDER_CQ_SHIFT, + udma_dev->caps.reorder_cq_shift); + } + } + udma_reg_write(&context->ext, QPCEX_RTT, QPCEX_RTT_INIT); + udma_reg_write(&context->ext, QPCEX_P_TYPE, QPCEX_P_TYPE_UDMA); +} + +static void edit_qpc_for_db(struct udma_qp_context *context, + struct udma_qp *qp) +{ + if (qp->en_flags & UDMA_QP_CAP_RQ_RECORD_DB) + udma_reg_enable(context, QPC_RQ_RECORD_EN); + + if (qp->en_flags & UDMA_QP_CAP_OWNER_DB) + udma_reg_enable(context, QPC_OWNER_MODE); +} + +static void edit_qpc_for_srqn(struct udma_qp *qp, + struct udma_qp_context *context) +{ + if (qp->qp_attr.jfr) { + udma_reg_enable(context, QPC_SRQ_EN); + udma_reg_write(context, QPC_SRQN, qp->qp_attr.jfr->jfrn); + } +} + +static void edit_qpc_for_rxcqn(struct udma_qp *qp, + struct udma_qp_context *context) +{ + if (qp->recv_jfc) + udma_reg_write(context, QPC_RX_CQN, qp->recv_jfc->cqn); +} + +static void edit_qpc_for_psn(const struct udma_modify_tp_attr *attr, + struct udma_qp_context *context) +{ + udma_reg_write(context, QPC_RX_REQ_EPSN, attr->rq_psn); + udma_reg_write(context, QPC_RAQ_PSN, attr->rq_psn - 1); + udma_reg_write(context, QPC_SQ_CUR_PSN, attr->sq_psn); + udma_reg_write(context, QPC_SQ_MAX_PSN, attr->sq_psn); +} + +static void edit_qpc_for_retransmission_parm(struct udma_dev *udma_device, + struct udma_qp *qp, + const struct udma_modify_tp_attr *attr, + struct udma_qp_context *context) +{ + if (qp->qp_type != QPT_UD) { + udma_reg_write(context, QPC_MIN_RNR_TIME, + attr->min_rnr_timer); + udma_reg_write(context, QPC_RETRY_CNT, + attr->retry_cnt); + udma_reg_write(context, QPC_RETRY_NUM_INIT, + attr->retry_cnt); + udma_reg_write(context, QPC_RNR_CNT, + attr->rnr_retry); + udma_reg_write(context, QPC_RNR_NUM_INIT, + attr->rnr_retry); + + if (check_qp_timeout_cfg_range(udma_device, &attr->ack_timeout)) + udma_reg_write(context, QPC_AT, attr->ack_timeout); + } +} + +static void edit_qpc_for_write(struct udma_qp *qp, + struct udma_qp_context *context) +{ + udma_reg_enable(context, QPC_FLUSH_EN); + udma_reg_enable(context, QPC_AW_EN); + udma_reg_enable(context, QPC_WN_EN); + udma_reg_enable(context, QPC_RMT_E2E); + udma_reg_write(context, QPC_SIG_TYPE, SIGNAL_REQ_WR); +} + +static void edit_qpc_for_receive(struct udma_qp *qp, + const struct udma_modify_tp_attr *attr, + struct udma_qp_context *context) +{ + uint8_t lp_pktn_ini; + uint8_t *dmac; + + udma_reg_write(context, QPC_DQPN, attr->dest_qp_num); + + dmac = (uint8_t *)attr->dmac; + memcpy(&context->dmac, dmac, sizeof(uint32_t)); + udma_reg_write(context, QPC_DMAC_L, *((uint32_t *)(&dmac[0]))); + udma_reg_write(context, QPC_DMAC_H, + *((uint16_t *)(&dmac[QPC_DMAC_H_IDX]))); + + qp->ubcore_path_mtu = get_mtu(qp, attr); + qp->path_mtu = to_udma_mtu(qp->ubcore_path_mtu); + udma_reg_write(context, QPC_MTU, qp->path_mtu); + + /* MTU * (2 ^ LP_PKTN_INI) shouldn't be bigger than 16KB */ + lp_pktn_ini = ilog2(MAX_LP_MSG_LEN / udma_mtu_enum_to_int(qp->path_mtu)); + + udma_reg_write(context, QPC_LP_PKTN_INI, lp_pktn_ini); + + /* ACK_REQ_FREQ should be larger than or equal to LP_PKTN_INI */ + udma_reg_write(context, QPC_ACK_REQ_FREQ, lp_pktn_ini); + + context->rq_rnr_timer = 0; + + /* rocee send 2^lp_sgen_ini segs every time */ + udma_reg_write(context, QPC_LP_SGEN_INI, SGEN_INI_VALUE); +} + +static int modify_qp_reset_to_rtr(struct udma_qp *qp, + const struct udma_modify_tp_attr *attr, + struct udma_qp_context *context, + struct udma_qp_context *context_mask) +{ + struct udma_dev *udma_device = qp->udma_device; + + udma_reg_write(context, QPC_RRE, 1); + udma_reg_write(context, QPC_RWE, 1); + + udma_reg_write(context, QPC_TST, qp->qp_type); + + udma_reg_write(context, QPC_RQWS, ilog2(qp->rq.max_gs)); + + /* No VLAN need to set 0xFFF */ + udma_reg_write(context, QPC_VLAN_ID, 0xfff); + + edit_qpc_for_db(context, qp); + + edit_qpc_for_inline(context, qp); + + edit_qpc_for_ext(context, qp, attr); + + edit_qpc_for_srqn(qp, context); + + edit_qpc_for_psn(attr, context); + + edit_qpc_for_retransmission_parm(udma_device, qp, attr, context); + + edit_qpc_for_rxcqn(qp, context); + + /* + * Enable atomic WRITE and persistence WRITE and Write With Notify + * operations in QPC when modify_qp_init_to_rtr. + */ + edit_qpc_for_write(qp, context); + + edit_qpc_for_receive(qp, attr, context); + + return 0; +} + +static int modify_qp_rtr_to_rts(struct udma_qp *qp, + struct udma_qp_context *context, + struct udma_qp_context *context_mask) +{ + struct udma_dev *udma_device = qp->udma_device; + int ret; + + qp->sq.wqe_offset = qp->sq.offset; + qp->sge.wqe_offset = qp->sge.offset; + + ret = config_qp_sq_buf(udma_device, qp, context, context_mask); + if (ret) { + dev_err(udma_device->dev, "failed to config sq buf, ret = %d.\n", + ret); + return ret; + } + if (qp->send_jfc) { + udma_reg_write(context, QPC_TX_CQN, qp->send_jfc->cqn); + udma_reg_clear(context_mask, QPC_TX_CQN); + } + + set_qpc_wqe_cnt(qp, context, context_mask); + + return 0; +} + +static int udma_set_opt_fields(struct udma_qp *qp, + const struct udma_modify_tp_attr *attr, + struct udma_qp_context *context, + struct udma_qp_context *context_mask) +{ + int ret = 0; + + if (attr == NULL) + goto out; + + if (attr->max_dest_rd_atomic) { + udma_reg_write(context, QPC_RR_MAX, + fls(attr->max_dest_rd_atomic - 1)); + udma_reg_clear(context_mask, QPC_RR_MAX); + } + + if (attr->max_rd_atomic) { + udma_reg_write(context, QPC_SR_MAX, + fls(attr->max_rd_atomic - 1)); + udma_reg_clear(context_mask, QPC_SR_MAX); + } + + udma_reg_write(context, QPC_RX_ACK_EPSN, attr->sq_psn); + udma_reg_write(context, QPC_RETRY_MSG_PSN_H, attr->sq_psn); + udma_reg_write(context, QPC_RETRY_NUM_INIT, attr->retry_cnt); + udma_reg_write(context, QPC_RETRY_MSG_FPKT_PSN, attr->sq_psn); + + udma_reg_clear(context_mask, QPC_RX_ACK_EPSN); + udma_reg_clear(context_mask, QPC_RETRY_MSG_PSN_H); + udma_reg_clear(context_mask, QPC_RETRY_NUM_INIT); + udma_reg_clear(context_mask, QPC_RETRY_MSG_FPKT_PSN); + +out: + return ret; +} + +static int udma_set_abs_fields(struct udma_qp *qp, + const struct udma_modify_tp_attr *attr, + enum udma_qp_state curr_state, + enum udma_qp_state new_state, + struct udma_qp_context *context, + struct udma_qp_context *context_mask) +{ + struct udma_dev *udma_device = qp->udma_device; + int ret = 0; + + if (curr_state == QPS_RESET && new_state == QPS_RTR) { + memset(context_mask, 0, udma_device->caps.qpc_sz); + ret = modify_qp_reset_to_rtr(qp, attr, context, context_mask); + if (ret) { + dev_err(udma_device->dev, + "Something went wrong during modify_qp_init_to_rtr\n"); + goto out; + } + } else if (curr_state == QPS_RESET && new_state == QPS_RTS) { + memset(context_mask, 0, udma_device->caps.qpc_sz); + ret = modify_qp_reset_to_rtr(qp, attr, context, context_mask); + if (ret) { + dev_err(udma_device->dev, + "Something went wrong during modify_qp_init_to_rtr\n"); + goto out; + } + ret = modify_qp_rtr_to_rts(qp, context, context_mask); + if (ret) { + dev_err(udma_device->dev, + "Something went wrong during modify_qp_rtr_to_rts\n"); + goto out; + } + } else if (curr_state == QPS_RTR && new_state == QPS_RTS) { + ret = modify_qp_rtr_to_rts(qp, context, context_mask); + if (ret) { + dev_err(udma_device->dev, + "Something went wrong during modify_qp_rtr_to_rts\n"); + goto out; + } + } + +out: + return ret; +} + +int udma_modify_qp_common(struct udma_qp *qp, + const struct udma_modify_tp_attr *attr, + enum udma_qp_state curr_state, + enum udma_qp_state new_state) +{ + struct udma_dev *udma_device = qp->udma_device; + struct udma_qp_context ctx[2] = {}; + struct udma_qp_context *context = ctx; + struct udma_qp_context *context_mask = ctx + 1; + int ret = 0; + + memset(context_mask, 0xff, udma_device->caps.qpc_sz); + if (new_state != QPS_RESET) { + ret = udma_set_abs_fields(qp, attr, curr_state, new_state, + context, context_mask); + if (ret) + goto out; + } + + ret = udma_set_opt_fields(qp, attr, context, context_mask); + if (ret) { + dev_err(udma_device->dev, "failed to set option fields, ret = %d.\n", + ret); + goto out; + } + + udma_reg_write(context, QPC_INV_CREDIT, (!!qp->qp_attr.jfr) ? 1 : 0); + udma_reg_clear(context_mask, QPC_INV_CREDIT); + /* Every status migrate must change state */ + udma_reg_write(context, QPC_QP_ST, new_state); + udma_reg_clear(context_mask, QPC_QP_ST); + + udma_set_path(attr, context, context_mask); + + udma_reg_write(&context->ext, QPCEX_P_TYPE, QPCEX_P_TYPE_UDMA); + udma_reg_clear(&context_mask->ext, QPCEX_P_TYPE); + /* SW pass context to HW */ + ret = udma_pass_qpc_to_hw(udma_device, context, context_mask, qp); + if (ret) { + dev_err(udma_device->dev, "failed to pass QPC to HW, ret = %d.\n", + ret); + goto out; + } + + qp->state = new_state; + +out: + return ret; +} + int fill_jfr_qp_attr(struct udma_dev *udma_dev, struct udma_qp_attr *qp_attr, struct udma_create_tp_ucmd *ucmd) { diff --git a/drivers/ub/hw/hns3/hns3_udma_qp.h b/drivers/ub/hw/hns3/hns3_udma_qp.h index ba4e604e5879..8e7bd5f8364c 100644 --- a/drivers/ub/hw/hns3/hns3_udma_qp.h +++ b/drivers/ub/hw/hns3/hns3_udma_qp.h @@ -18,6 +18,161 @@ #include "hns3_udma_device.h" +#define UDMA_GID_SIZE 16 +#define SGEN_INI_VALUE 3 +#define QP_TIMEOUT_MAX 31 +#define WQE_SGE_BA_OFFSET 3 +#define H_ADDR_OFFSET 32 +#define MAX_LP_MSG_LEN 16384 +#define QPC_DMAC_H_IDX 4 +#define UDP_RANGE_BASE 8 +#define UDMA_SQ_WQE_SHIFT 6 + +struct udma_qp_context_ex { + uint32_t data[64]; +}; + +struct udma_qp_context { + uint32_t qpc_context1; + uint32_t wqe_sge_ba; + uint32_t qpc_context2[5]; + uint8_t dgid[UDMA_GID_SIZE]; + uint32_t dmac; + uint32_t qpc_context3[3]; + uint32_t qkey_xrcd; + uint32_t qpc_context4[11]; + uint32_t rq_rnr_timer; + uint32_t qpc_context5[36]; + struct udma_qp_context_ex ext; +}; + +#define QPC_FIELD_LOC(h, l) ((uint64_t)(h) << 32 | (l)) + +#define QPC_TST QPC_FIELD_LOC(2, 0) +#define QPC_SGE_SHIFT QPC_FIELD_LOC(7, 3) +#define QPC_WQE_SGE_BA_L QPC_FIELD_LOC(63, 32) +#define QPC_WQE_SGE_BA_H QPC_FIELD_LOC(92, 64) +#define QPC_SQ_HOP_NUM QPC_FIELD_LOC(94, 93) +#define QPC_WQE_SGE_BA_PG_SZ QPC_FIELD_LOC(99, 96) +#define QPC_WQE_SGE_BUF_PG_SZ QPC_FIELD_LOC(103, 100) +#define QPC_SGE_HOP_NUM QPC_FIELD_LOC(131, 130) +#define QPC_RQWS QPC_FIELD_LOC(135, 132) +#define QPC_SQ_SHIFT QPC_FIELD_LOC(139, 136) +#define QPC_GMV_IDX QPC_FIELD_LOC(159, 144) +#define QPC_HOPLIMIT QPC_FIELD_LOC(167, 160) +#define QPC_VLAN_ID QPC_FIELD_LOC(187, 176) +#define QPC_MTU QPC_FIELD_LOC(191, 188) +#define QPC_FL QPC_FIELD_LOC(211, 192) +#define QPC_SL QPC_FIELD_LOC(215, 212) +#define QPC_AT QPC_FIELD_LOC(223, 219) +#define QPC_DMAC_L QPC_FIELD_LOC(383, 352) +#define QPC_DMAC_H QPC_FIELD_LOC(399, 384) +#define QPC_DQPN QPC_FIELD_LOC(439, 416) +#define QPC_LP_PKTN_INI QPC_FIELD_LOC(447, 444) +#define QPC_CONGEST_ALGO_TMPL_ID QPC_FIELD_LOC(455, 448) +#define QPC_QP_ST QPC_FIELD_LOC(479, 477) +#define QPC_RQ_RECORD_EN QPC_FIELD_LOC(512, 512) +#define QPC_RQ_DB_RECORD_ADDR_L QPC_FIELD_LOC(543, 513) +#define QPC_RQ_DB_RECORD_ADDR_H QPC_FIELD_LOC(575, 544) +#define QPC_SRQN QPC_FIELD_LOC(599, 576) +#define QPC_SRQ_EN QPC_FIELD_LOC(600, 600) +#define QPC_RRE QPC_FIELD_LOC(601, 601) +#define QPC_RWE QPC_FIELD_LOC(602, 602) +#define QPC_RX_CQN QPC_FIELD_LOC(631, 608) +#define QPC_XRC_QP_TYPE QPC_FIELD_LOC(632, 632) +#define QPC_CQEIE QPC_FIELD_LOC(633, 633) +#define QPC_MIN_RNR_TIME QPC_FIELD_LOC(639, 635) +#define QPC_FLUSH_EN QPC_FIELD_LOC(821, 821) +#define QPC_AW_EN QPC_FIELD_LOC(822, 822) +#define QPC_WN_EN QPC_FIELD_LOC(823, 823) +#define QPC_INV_CREDIT QPC_FIELD_LOC(832, 832) +#define QPC_RX_REQ_EPSN QPC_FIELD_LOC(863, 840) +#define QPC_RR_MAX QPC_FIELD_LOC(1102, 1100) +#define QPC_RAQ_PSN QPC_FIELD_LOC(1207, 1184) +#define QPC_SQ_PRODUCER_IDX QPC_FIELD_LOC(1263, 1248) +#define QPC_SQ_CONSUMER_IDX QPC_FIELD_LOC(1279, 1264) +#define QPC_SQ_CUR_BLK_ADDR_L QPC_FIELD_LOC(1311, 1280) +#define QPC_SQ_CUR_BLK_ADDR_H QPC_FIELD_LOC(1331, 1312) +#define QPC_LP_SGEN_INI QPC_FIELD_LOC(1335, 1334) +#define QPC_ACK_REQ_FREQ QPC_FIELD_LOC(1349, 1344) +#define QPC_SQ_CUR_PSN QPC_FIELD_LOC(1375, 1352) +#define QPC_SQ_CUR_SGE_BLK_ADDR_L QPC_FIELD_LOC(1439, 1408) +#define QPC_SQ_CUR_SGE_BLK_ADDR_H QPC_FIELD_LOC(1459, 1440) +#define QPC_OWNER_MODE QPC_FIELD_LOC(1536, 1536) +#define QPC_SQ_MAX_PSN QPC_FIELD_LOC(1567, 1544) +#define QPC_RMT_E2E QPC_FIELD_LOC(1660, 1660) +#define QPC_SR_MAX QPC_FIELD_LOC(1663, 1661) +#define QPC_RETRY_NUM_INIT QPC_FIELD_LOC(1690, 1688) +#define QPC_RETRY_CNT QPC_FIELD_LOC(1695, 1693) +#define QPC_RETRY_MSG_MSN QPC_FIELD_LOC(1743, 1728) +#define QPC_RETRY_MSG_PSN_H QPC_FIELD_LOC(1767, 1760) +#define QPC_RETRY_MSG_FPKT_PSN QPC_FIELD_LOC(1791, 1768) +#define QPC_RX_SQ_CUR_BLK_ADDR_L QPC_FIELD_LOC(1823, 1792) +#define QPC_RX_SQ_CUR_BLK_ADDR_H QPC_FIELD_LOC(1843, 1824) +#define QPC_RX_ACK_EPSN QPC_FIELD_LOC(1943, 1920) +#define QPC_RNR_NUM_INIT QPC_FIELD_LOC(1946, 1944) +#define QPC_RNR_CNT QPC_FIELD_LOC(1949, 1947) +#define QPC_TX_CQN QPC_FIELD_LOC(2007, 1984) +#define QPC_SIG_TYPE QPC_FIELD_LOC(2008, 2008) + +#define QPCEX_FIELD_LOC(h, l) ((uint64_t)(h) << 32 | (l)) + +#define QPCEX_ON_FLIGHT_SIZE_H_SHIFT 3 +#define QPCEX_REORDER_CQ_ADDR_SHIFT 12 +#define QPCEX_DATA_UDP_SRCPORT_H_SHIFT 11 +#define QPCEX_RTT_INIT 100 +#define QPCEX_P_TYPE_UDMA 0x1 +#define MAX_SERVICE_LEVEL 0x7 + +#define QPCEX_DIP_CTX_IDX_VLD QPCEX_FIELD_LOC(2, 2) +#define QPCEX_DIP_CTX_IDX QPCEX_FIELD_LOC(22, 3) +#define QPCEX_SQ_RQ_NOT_FORBID_EN QPCEX_FIELD_LOC(23, 23) +#define QPCEX_RTT QPCEX_FIELD_LOC(81, 66) +#define QPCEX_AR_EN QPCEX_FIELD_LOC(112, 112) +#define QPCEX_UDP_SRCPORT_RANGE QPCEX_FIELD_LOC(116, 113) +#define QPCEX_DATA_UDP_SRCPORT_L QPCEX_FIELD_LOC(127, 117) +#define QPCEX_DATA_UDP_SRCPORT_H QPCEX_FIELD_LOC(132, 128) +#define QPCEX_ACK_UDP_SRCPORT QPCEX_FIELD_LOC(148, 133) +#define QPCEX_REORDER_CAP QPCEX_FIELD_LOC(155, 153) +#define QPCEX_ON_FLIGHT_SIZE_L QPCEX_FIELD_LOC(159, 157) +#define QPCEX_ON_FLIGHT_SIZE_H QPCEX_FIELD_LOC(172, 160) +#define QPCEX_OOR_EN QPCEX_FIELD_LOC(266, 266) +#define QPCEX_P_TYPE QPCEX_FIELD_LOC(268, 268) +#define QPCEX_DYN_AT QPCEX_FIELD_LOC(272, 269) +#define QPCEX_DEID_H QPCEX_FIELD_LOC(319, 288) +#define QPCEX_REORDER_CQ_EN QPCEX_FIELD_LOC(427, 427) +#define QPCEX_REORDER_CQ_ADDR_L QPCEX_FIELD_LOC(447, 428) +#define QPCEX_REORDER_CQ_ADDR_H QPCEX_FIELD_LOC(479, 448) +#define QPCEX_REORDER_CQ_SHIFT QPCEX_FIELD_LOC(483, 480) +#define UDP_SRCPORT_RANGE_BASE 7 +#define UDP_SRCPORT_RANGE_SIZE_MASK 0xF + +struct udma_modify_tp_attr { + uint8_t dmac[UBCORE_MAC_BYTES]; + uint32_t dest_qp_num; + uint8_t retry_cnt; + uint8_t rnr_retry; + uint8_t priority; + uint8_t ack_timeout; + uint32_t sq_psn; + uint32_t rq_psn; + uint8_t max_dest_rd_atomic; + uint8_t max_rd_atomic; + int qp_access_flags; + uint8_t min_rnr_timer; + uint32_t qkey; + enum ubcore_mtu path_mtu; + uint8_t hop_limit; + uint8_t dgid[UDMA_GID_SIZE]; + uint8_t dipv4[4]; + uint8_t sgid_index; + uint16_t data_udp_start; + uint16_t ack_udp_start; + uint16_t udp_range; + uint32_t ar_en; + enum udma_cong_type cong_alg; +}; + struct udma_qp_cap { uint32_t max_send_wr; uint32_t max_recv_wr; @@ -45,6 +200,7 @@ struct udma_qp_attr { struct ubcore_ucontext *uctx; struct udma_jfc *send_jfc; struct udma_jfc *recv_jfc; + struct udma_jfs *jfs; struct udma_jfr *jfr; struct udma_qp_cap cap; enum udma_qp_type qp_type; @@ -78,6 +234,7 @@ struct udma_qp { struct udma_dev *udma_device; enum udma_qp_type qp_type; struct udma_qp_attr qp_attr; + struct udma_wq sq; struct udma_wq rq; struct udma_jfc *send_jfc; struct udma_jfc *recv_jfc; @@ -91,7 +248,10 @@ struct udma_qp { refcount_t refcount; struct completion free; struct udma_qp_sge sge; + enum udma_mtu path_mtu; + enum ubcore_mtu ubcore_path_mtu; uint32_t max_inline_data; + uint8_t sl; struct list_head node; /* all qps are on a list */ struct list_head rq_node; /* all recv qps are on a list */ struct list_head sq_node; /* all send qps are on a list */ @@ -104,6 +264,10 @@ struct udma_qp { #define gen_qpn(high, mid, low) ((high) | (mid) | (low)) +int udma_modify_qp_common(struct udma_qp *qp, + const struct udma_modify_tp_attr *attr, + enum udma_qp_state curr_state, + enum udma_qp_state new_state); int udma_fill_qp_attr(struct udma_dev *udma_dev, struct udma_qp_attr *qp_attr, const struct ubcore_tp_cfg *cfg, struct ubcore_udata *udata); int udma_create_qp_common(struct udma_dev *udma_dev, struct udma_qp *qp, diff --git a/drivers/ub/hw/hns3/hns3_udma_tp.h b/drivers/ub/hw/hns3/hns3_udma_tp.h index 7355447a3736..9832ac0978c5 100644 --- a/drivers/ub/hw/hns3/hns3_udma_tp.h +++ b/drivers/ub/hw/hns3/hns3_udma_tp.h @@ -19,6 +19,8 @@ #include #include "hns3_udma_qp.h" +#define MAX_HOP_LIMIT 255 + struct udma_tp { struct ubcore_tp ubcore_tp; struct udma_qp qp; -- Gitee From dcca9f02120681affd0882544848f2733f9e61ea Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Wed, 30 Aug 2023 15:40:40 +0800 Subject: [PATCH 29/42] hns3 udma: support modify tp and destroy tp driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA --------------------------------------------------------------------- URMA uses the modify_tp function to modify the attributes of tp. UDMA driver will update qp context to hardware based on the value of mask. Signed-off-by: Yu Zhang --- drivers/ub/hw/hns3/hns3_udma_main.c | 5 + drivers/ub/hw/hns3/hns3_udma_tp.c | 175 +++++++++++++++++++++++++++- drivers/ub/hw/hns3/hns3_udma_tp.h | 8 ++ 3 files changed, 185 insertions(+), 3 deletions(-) diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c index 2109f82ef437..bb78d0f2587a 100644 --- a/drivers/ub/hw/hns3/hns3_udma_main.c +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -20,11 +20,13 @@ #include "hnae3.h" #include "hns3_udma_device.h" #include "hns3_udma_hem.h" +#include "hns3_udma_tp.h" #include "hns3_udma_jfr.h" #include "hns3_udma_jfc.h" #include "hns3_udma_jfs.h" #include "hns3_udma_segment.h" #include "hns3_udma_cmd.h" + static int udma_set_eid(struct ubcore_device *dev, union ubcore_eid eid) { struct udma_dev *udma_dev = to_udma_dev(dev); @@ -328,6 +330,9 @@ static struct ubcore_ops g_udma_dev_ops = { .destroy_jfr = udma_destroy_jfr, .import_jfr = udma_import_jfr, .unimport_jfr = udma_unimport_jfr, + .create_tp = udma_create_tp, + .modify_tp = udma_modify_tp, + .destroy_tp = udma_destroy_tp, }; static void udma_cleanup_uar_table(struct udma_dev *dev) diff --git a/drivers/ub/hw/hns3/hns3_udma_tp.c b/drivers/ub/hw/hns3/hns3_udma_tp.c index ea765a6bd7e8..fa9eb5e06478 100644 --- a/drivers/ub/hw/hns3/hns3_udma_tp.c +++ b/drivers/ub/hw/hns3/hns3_udma_tp.c @@ -16,8 +16,168 @@ #include #include #include "hns3_udma_jfr.h" +#include "hns3_udma_jfs.h" #include "hns3_udma_tp.h" +static enum udma_qp_state to_udma_qp_state(enum ubcore_tp_state state) +{ + switch (state) { + case UBCORE_TP_STATE_RESET: + return QPS_RESET; + case UBCORE_TP_STATE_RTR: + return QPS_RTR; + case UBCORE_TP_STATE_RTS: + return QPS_RTS; + case UBCORE_TP_STATE_ERROR: + return QPS_ERR; + default: + return QPS_ERR; + } +} + +struct udma_modify_tp_attr *udma_get_m_attr(struct ubcore_tp *tp, struct udma_qp *qp, + const struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask mask) +{ + struct udma_modify_tp_attr *m_attr; + + m_attr = kzalloc(sizeof(*m_attr), GFP_KERNEL); + if (m_attr == NULL) + return NULL; + + memcpy(m_attr->dmac, tp->peer_net_addr.mac, sizeof(m_attr->dmac)); + + m_attr->max_dest_rd_atomic = 0; + m_attr->max_rd_atomic = 0; + m_attr->rq_psn = tp->rx_psn; + m_attr->hop_limit = MAX_HOP_LIMIT; + m_attr->sgid_index = 0; + *(uint32_t *)(&m_attr->dipv4) = *(uint32_t *)(tp->peer_eid.raw + + SGID_H_SHIFT); + memcpy(m_attr->dgid, tp->peer_eid.raw, sizeof(tp->peer_eid.raw)); + + if (!qp->qp_attr.is_tgt) { + m_attr->retry_cnt = qp->retry_cnt; + m_attr->ack_timeout = qp->ack_timeout; + m_attr->rnr_retry = qp->rnr_retry; + m_attr->priority = qp->priority; + } else { + m_attr->min_rnr_timer = qp->min_rnr_timer; + } + + if (mask.bs.peer_tpn) + m_attr->dest_qp_num = attr->peer_tpn; + + if (mask.bs.tx_psn) + m_attr->sq_psn = attr->tx_psn; + else + m_attr->sq_psn = tp->tx_psn; + + if (mask.bs.mtu) + m_attr->path_mtu = attr->mtu; + + return m_attr; +} + +int udma_modify_tp(struct ubcore_tp *tp, const struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask mask) +{ + struct udma_modify_tp_attr *m_attr; + enum udma_qp_state target_state; + enum udma_qp_state curr_state; + struct udma_dev *udma_device; + struct udma_tp *utp; + struct udma_qp *qp; + int ret = -EINVAL; + + udma_device = to_udma_dev(tp->ub_dev); + utp = to_udma_tp(tp); + + qp = &utp->qp; + if (!qp) + goto error; + + curr_state = to_udma_qp_state(tp->state); + + if (mask.bs.state) + target_state = to_udma_qp_state(attr->state); + else + target_state = QPS_ERR; + + m_attr = udma_get_m_attr(tp, qp, attr, mask); + if (!m_attr) + goto error; + + qp->udma_device = udma_device; + qp->send_jfc = qp->qp_attr.send_jfc; + qp->recv_jfc = qp->qp_attr.recv_jfc; + if (attr) + qp->ubcore_path_mtu = attr->mtu; + ret = udma_modify_qp_common(qp, m_attr, curr_state, target_state); + kfree(m_attr); +error: + return ret; +} + +void *udma_erase_tp(struct udma_tp *udma_tp) +{ + struct udma_qp_attr *qp_attr; + struct udma_jfr *jfr; + struct udma_jfs *jfs; + + qp_attr = &udma_tp->qp.qp_attr; + jfr = qp_attr->jfr; + jfs = qp_attr->jfs; + + if (jfr) + return xa_erase(&jfr->tp_table_xa, + udma_tp->ubcore_tp.tpn); + + if (jfs) + return xa_erase(&jfs->node_table, + udma_tp->ubcore_tp.tpn); + + return udma_tp; +} + +int udma_destroy_tp(struct ubcore_tp *tp) +{ + struct udma_dev *udma_device = to_udma_dev(tp->ub_dev); + enum udma_qp_state curr_state; + struct udma_tp *udma_tp; + struct udma_qp *qp; + int ret = 0; + + udma_tp = to_udma_tp(tp); + if (!udma_erase_tp(udma_tp)) { + dev_err(udma_device->dev, + "failed to find tp, tpn = 0x%x\n", tp->tpn); + return 0; + } + + qp = &udma_tp->qp; + curr_state = to_udma_qp_state(tp->state); + + if (qp->state != QPS_RESET) { + ret = udma_modify_qp_common(qp, NULL, curr_state, QPS_RESET); + if (ret) { + dev_err(udma_device->dev, + "Modify QP 0x%06llx to Reset failed(%d).\n", + qp->qpn, ret); + goto error; + } + } + + udma_destroy_qp_common(udma_device, qp); + + kfree(udma_tp); + + return ret; + +error: + return -EINVAL; +} + static void udma_set_tp(struct ubcore_device *dev, const struct ubcore_tp_cfg *cfg, struct udma_tp *tp) { @@ -48,19 +208,28 @@ static int udma_store_tp(struct udma_dev *udma_device, struct udma_tp *tp, { struct udma_qp_attr *qp_attr; struct udma_jfr *jfr; + struct udma_jfs *jfs; int ret = 0; qp_attr = &tp->qp.qp_attr; jfr = qp_attr->jfr; + jfs = qp_attr->jfs; if (jfr) { - ret = xa_err(xa_store(&jfr->tp_table_xa, - tp->ubcore_tp.tpn, tp, - GFP_KERNEL)); + ret = xa_err(xa_store(&jfr->tp_table_xa, tp->ubcore_tp.tpn, tp, + GFP_KERNEL)); if (ret) { dev_err(udma_device->dev, "failed store jfr tp, ret = %d\n", ret); return ret; } + } else if (jfs) { + ret = xa_err(xa_store(&jfs->node_table, tp->ubcore_tp.tpn, tp, + GFP_KERNEL)); + if (ret) { + dev_err(udma_device->dev, + "failed store jfs tp, ret = %d\n", ret); + return ret; + } } return ret; diff --git a/drivers/ub/hw/hns3/hns3_udma_tp.h b/drivers/ub/hw/hns3/hns3_udma_tp.h index 9832ac0978c5..90fa1f27b881 100644 --- a/drivers/ub/hw/hns3/hns3_udma_tp.h +++ b/drivers/ub/hw/hns3/hns3_udma_tp.h @@ -26,9 +26,17 @@ struct udma_tp { struct udma_qp qp; struct ubcore_jetty_id tjetty_id; }; + +static inline struct udma_tp *to_udma_tp(struct ubcore_tp *ubcore_tp) +{ + return container_of(ubcore_tp, struct udma_tp, ubcore_tp); +} + struct ubcore_tp *udma_create_tp(struct ubcore_device *dev, const struct ubcore_tp_cfg *cfg, struct ubcore_udata *udata); int udma_destroy_tp(struct ubcore_tp *tp); +int udma_modify_tp(struct ubcore_tp *tp, const struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask mask); #endif /* _UDMA_TP_H */ -- Gitee From 85edae91cba9c51c5651fd404d570be73eae1216 Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Wed, 30 Aug 2023 16:41:01 +0800 Subject: [PATCH 30/42] hns3 udma: add support of UM JFR driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA --------------------------------------------------------------------- In UM(Unreliable Message) mode, the creation of the UM JFR requires a QP in the create_jfr phase. Signed-off-by: Yu Zhang --- drivers/ub/hw/hns3/hns3_udma_jfr.c | 77 ++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/drivers/ub/hw/hns3/hns3_udma_jfr.c b/drivers/ub/hw/hns3/hns3_udma_jfr.c index a4111b5083bb..a9d4d600b808 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jfr.c +++ b/drivers/ub/hw/hns3/hns3_udma_jfr.c @@ -392,6 +392,73 @@ static void free_jfr_buf(struct udma_dev *dev, struct udma_jfr *jfr) free_jfr_idx(dev, jfr); } +static int udma_modify_jfr_um_qpc(struct udma_dev *dev, struct udma_jfr *jfr, + enum udma_qp_state target_state) +{ + struct udma_modify_tp_attr attr = {}; + struct udma_qp *qp = jfr->um_qp; + int ret; + + attr.path_mtu = UBCORE_MTU_4096; + qp->udma_device = dev; + qp->qp_attr.jfr = jfr; + qp->recv_jfc = to_udma_jfc(jfr->ubcore_jfr.jfr_cfg.jfc); + qp->send_jfc = NULL; + ret = udma_modify_qp_common(qp, &attr, jfr->um_qp->state, target_state); + if (ret) + dev_err(dev->dev, "failed to modify qpc to RTR.\n"); + + qp->state = target_state; + return ret; +} + +static int alloc_jfr_um_qp(struct udma_dev *dev, struct udma_jfr *jfr) +{ + struct ubcore_udata udata; + struct udma_qp *qp; + int ret; + + qp = kzalloc(sizeof(*qp), GFP_KERNEL); + if (!qp) + return -ENOMEM; + + qp->qp_type = QPT_UD; + qp->qp_attr.qp_type = QPT_UD; + qp->qp_attr.qpn_map = &jfr->qpn_map; + qp->qp_attr.recv_jfc = to_udma_jfc(jfr->ubcore_jfr.jfr_cfg.jfc); + qp->qp_attr.send_jfc = NULL; + udata.uctx = NULL; + ret = udma_create_qp_common(dev, qp, &udata); + if (ret) { + dev_err(dev->dev, "failed to create qpc.\n"); + goto failed_create_qpc; + } + jfr->um_qp = qp; + + qp->state = QPS_RESET; + ret = udma_modify_jfr_um_qpc(dev, jfr, QPS_RTR); + if (ret) { + dev_err(dev->dev, "failed to modify qpc.\n"); + goto failed_modify_qpc; + } + + return 0; + +failed_modify_qpc: + udma_destroy_qp_common(dev, qp); +failed_create_qpc: + kfree(qp); + + return ret; +} + +void destroy_jfr_um_qp(struct udma_dev *dev, struct udma_jfr *jfr) +{ + udma_modify_jfr_um_qpc(dev, jfr, QPS_RESET); + udma_destroy_qp_common(dev, jfr->um_qp); + kfree(jfr->um_qp); +} + struct ubcore_jfr *udma_create_jfr(struct ubcore_device *dev, const struct ubcore_jfr_cfg *cfg, struct ubcore_udata *udata) { @@ -419,8 +486,16 @@ struct ubcore_jfr *udma_create_jfr(struct ubcore_device *dev, const struct ubcor init_jetty_x_qpn_bitmap(udma_dev, &jfr->qpn_map, udma_dev->caps.num_jfr_shift, UDMA_JFR_QPN_PREFIX, jfr->jfrn); + if (cfg->trans_mode == UBCORE_TP_UM) { + ret = alloc_jfr_um_qp(udma_dev, jfr); + if (ret) + goto err_alloc_jfrc; + } return &jfr->ubcore_jfr; +err_alloc_jfrc: + clean_jetty_x_qpn_bitmap(&jfr->qpn_map); + free_jfrc(udma_dev, jfr->jfrn); err_alloc_buf: free_jfr_buf(udma_dev, jfr); err_alloc_jfr: @@ -434,6 +509,8 @@ int udma_destroy_jfr(struct ubcore_jfr *jfr) struct udma_dev *dev = to_udma_dev(jfr->ub_dev); struct udma_jfr *udma_jfr = to_udma_jfr(jfr); + if (udma_jfr->um_qp) + destroy_jfr_um_qp(dev, udma_jfr); clean_jetty_x_qpn_bitmap(&udma_jfr->qpn_map); free_jfrc(dev, udma_jfr->jfrn); free_jfr_buf(dev, udma_jfr); -- Gitee From 4888db6edae7e3cff095d044dbdd680a13cab302 Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Thu, 31 Aug 2023 19:23:40 +0800 Subject: [PATCH 31/42] hns3 udma: support create and destroy JETTY driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA --------------------------------------------------------------------- This patch supports the creation and deletion of JETTY. Compared with the JFS/JFR programming mode, JETTY provides both sending and receiving functions. In the sending direction, JETTY contains a JFS that supports sending messages to multiple destination nodes. In the receive direction, JETTY contains a JFR that can receive data from different nodes. JFS and JFR in JETTY can point to different JFCs. When creating a JETTY, the depth and attributes of the JFS and JFR contained in the JETTY need to specified. UDMA driver will allcate memory resources and configure the hardware(QP Context and SRQ Context). Signed-off-by: Yu Zhang --- drivers/ub/hw/hns3/Makefile | 3 +- drivers/ub/hw/hns3/hns3_udma_abi.h | 12 + drivers/ub/hw/hns3/hns3_udma_device.h | 9 +- drivers/ub/hw/hns3/hns3_udma_hem.c | 9 - drivers/ub/hw/hns3/hns3_udma_jetty.c | 378 +++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_jetty.h | 60 ++++ drivers/ub/hw/hns3/hns3_udma_jfr.c | 14 + drivers/ub/hw/hns3/hns3_udma_jfr.h | 1 + drivers/ub/hw/hns3/hns3_udma_main.c | 3 + drivers/ub/hw/hns3/hns3_udma_qp.c | 444 ++++++++++++++++++++++++- drivers/ub/hw/hns3/hns3_udma_qp.h | 7 + drivers/ub/hw/hns3/hns3_udma_segment.c | 1 - drivers/ub/hw/hns3/hns3_udma_tp.c | 174 +++++++++- drivers/ub/hw/hns3/hns3_udma_tp.h | 5 + 14 files changed, 1083 insertions(+), 37 deletions(-) create mode 100644 drivers/ub/hw/hns3/hns3_udma_jetty.c create mode 100644 drivers/ub/hw/hns3/hns3_udma_jetty.h diff --git a/drivers/ub/hw/hns3/Makefile b/drivers/ub/hw/hns3/Makefile index 38ab0ff041d5..1d1452aad8d9 100644 --- a/drivers/ub/hw/hns3/Makefile +++ b/drivers/ub/hw/hns3/Makefile @@ -10,6 +10,7 @@ ccflags-y += -I$(srctree)/drivers/net/ethernet/hisilicon/hns3/ \ $(MODULE_NAME)-objs := hns3_udma_hw.o hns3_udma_main.o hns3_udma_cmd.o \ hns3_udma_hem.o hns3_udma_qp.o hns3_udma_eq.o \ hns3_udma_db.o hns3_udma_jfc.o hns3_udma_jfr.o \ - hns3_udma_segment.o hns3_udma_tp.o hns3_udma_jfs.o + hns3_udma_segment.o hns3_udma_tp.o hns3_udma_jfs.o \ + hns3_udma_jetty.o obj-$(CONFIG_UB_UDMA_HNS3) := hns3_udma.o diff --git a/drivers/ub/hw/hns3/hns3_udma_abi.h b/drivers/ub/hw/hns3/hns3_udma_abi.h index 15d60481e37c..9c6628fba99b 100644 --- a/drivers/ub/hw/hns3/hns3_udma_abi.h +++ b/drivers/ub/hw/hns3/hns3_udma_abi.h @@ -25,6 +25,7 @@ #define UDMA_JETTY_X_PREFIX_BIT_NUM 2 #define UDMA_JFS_QPN_PREFIX 0x0 #define UDMA_JFR_QPN_PREFIX 0x1 +#define UDMA_JETTY_QPN_PREFIX 0x2 #define UDMA_ADDR_4K_MASK 0xfffUL #define URMA_SEG_ACCESS_GUARD (1UL << 5) @@ -99,6 +100,13 @@ struct udma_create_tp_ucmd { uint64_t sdb_addr; }; +struct udma_create_jetty_ucmd { + struct udma_create_tp_ucmd create_tp_ucmd; + uint32_t jfr_id; + uint64_t buf_addr; + uint64_t sdb_addr; +}; + enum udma_qp_cap_flags { UDMA_QP_CAP_RQ_RECORD_DB = 1 << 0, UDMA_QP_CAP_SQ_RECORD_DB = 1 << 1, @@ -113,6 +121,10 @@ struct udma_create_tp_resp { uint8_t priority; }; +struct udma_create_jetty_resp { + struct udma_create_tp_resp create_tp_resp; +}; + struct udma_create_jfs_ucmd { struct udma_create_tp_ucmd create_tp_ucmd; }; diff --git a/drivers/ub/hw/hns3/hns3_udma_device.h b/drivers/ub/hw/hns3/hns3_udma_device.h index 295cc5125e15..e035dadc5965 100644 --- a/drivers/ub/hw/hns3/hns3_udma_device.h +++ b/drivers/ub/hw/hns3/hns3_udma_device.h @@ -45,6 +45,8 @@ #define UDMA_GMV_ENTRY_SZ 32 #define UDMA_CQ_BANK_NUM 4 + +#define UDMA_SGE_IN_WQE 2 #define UDMA_SGE_SHIFT 4 #define UDMA_SGE_SIZE 16 #define UDMA_IDX_QUE_ENTRY_SZ 4 @@ -53,6 +55,9 @@ #define UDMA_PAGE_SIZE (1 << UDMA_HW_PAGE_SHIFT) #define udma_hw_page_align(x) ALIGN(x, 1 << UDMA_HW_PAGE_SHIFT) +#define UDMA_DWQE_SIZE 65536 +#define UDMA_DWQE_MMAP_QP_NUM 1024 + #define UDMA_HOP_NUM_0 0xff #define UDMA_CAP_FLAGS_EX_SHIFT 12 @@ -137,6 +142,8 @@ #define UDMA_DEV_START_OFFSET 2 #define UDMA_DEV_EX_START_OFFSET 4 +#define UDMA_MIN_JFS_DEPTH 64 + enum { NO_ARMED = 0x0 }; @@ -287,8 +294,6 @@ struct udma_buf_attr { } region[UDMA_MAX_BT_REGION]; uint32_t region_count; /* valid region count */ uint32_t page_shift; /* buffer page shift */ - /* only alloc buffer-required MTT memory */ - bool mtt_only; }; struct udma_buf_list { diff --git a/drivers/ub/hw/hns3/hns3_udma_hem.c b/drivers/ub/hw/hns3/hns3_udma_hem.c index e425efaa258f..7950c82306bb 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hem.c +++ b/drivers/ub/hw/hns3/hns3_udma_hem.c @@ -1920,15 +1920,6 @@ int udma_mtr_create(struct udma_dev *udma_dev, struct udma_mtr *mtr, return ret; } - /* The caller has its own buffer list and invokes the udma_mtr_map() - * to finish the MTT configuration. - */ - if (buf_attr->mtt_only) { - mtr->umem = NULL; - mtr->kmem = NULL; - return 0; - } - ret = mtr_alloc_bufs(udma_dev, mtr, buf_attr, user_addr, is_user); if (ret) { dev_err(dev, "failed to alloc mtr bufs, ret = %d.\n", ret); diff --git a/drivers/ub/hw/hns3/hns3_udma_jetty.c b/drivers/ub/hw/hns3/hns3_udma_jetty.c new file mode 100644 index 000000000000..c9cdb22d328e --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_jetty.c @@ -0,0 +1,378 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include +#include "urma/ubcore_types.h" +#include "hns3_udma_tp.h" +#include "hns3_udma_jfc.h" +#include "hns3_udma_jfr.h" +#include "hns3_udma_db.h" +#include "hns3_udma_jetty.h" + +static void init_jetty_cfg(struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg) +{ + jetty->shared_jfr = cfg->flag.bs.share_jfr; + jetty->tp_mode = cfg->trans_mode; + jetty->ubcore_jetty.jetty_cfg = *cfg; +} + +static void udma_fill_jetty_um_qp_attr(struct udma_dev *dev, + struct udma_jetty *jetty, + struct ubcore_ucontext *uctx, + const struct ubcore_jetty_cfg *cfg) +{ + struct udma_ucontext *udma_ctx = to_udma_ucontext(uctx); + struct udma_qp_attr *qp_attr = &jetty->qp.qp_attr; + + qp_attr->is_jetty = true; + qp_attr->uctx = uctx; + qp_attr->pdn = udma_ctx->pdn; + qp_attr->send_jfc = to_udma_jfc(cfg->send_jfc); + qp_attr->jetty = jetty; + qp_attr->cap.max_send_wr = cfg->jfs_depth; + qp_attr->cap.max_send_sge = cfg->max_send_sge; + qp_attr->cap.max_inline_data = cfg->max_inline_data; + qp_attr->cap.retry_cnt = cfg->retry_cnt; + qp_attr->cap.rnr_retry = cfg->rnr_retry; + qp_attr->cap.ack_timeout = cfg->err_timeout; + qp_attr->qp_type = QPT_UD; + + qp_attr->jfr = jetty->udma_jfr; + qp_attr->qpn_map = &jetty->qpn_map; + qp_attr->recv_jfc = to_udma_jfc(cfg->recv_jfc); + if (jetty->ubcore_jetty.jetty_cfg.priority >= dev->caps.sl_num) { + qp_attr->priority = + dev->caps.sl_num > 0 ? dev->caps.sl_num - 1 : 0; + dev_err(dev->dev, "The setted priority (%d) should smaller than the max priority (%d), priority (%d) is used\n", + jetty->ubcore_jetty.jetty_cfg.priority, + dev->caps.sl_num, qp_attr->priority); + } else { + qp_attr->priority = jetty->ubcore_jetty.jetty_cfg.priority; + } +} + +static int udma_modify_qp_jetty(struct udma_dev *dev, struct udma_jetty *jetty, + enum udma_qp_state target_state) +{ + struct udma_modify_tp_attr m_attr = {}; + struct udma_qp *qp; + int ret; + + qp = &jetty->qp; + qp->udma_device = dev; + qp->send_jfc = qp->qp_attr.send_jfc; + qp->recv_jfc = qp->qp_attr.recv_jfc; + + m_attr.path_mtu = UBCORE_MTU_4096; + m_attr.hop_limit = MAX_HOP_LIMIT; + + ret = udma_modify_qp_common(qp, &m_attr, jetty->qp.state, target_state); + if (ret) + dev_err(dev->dev, "failed to modify qpc to RTS in Jetty.\n"); + + qp->state = target_state; + return ret; +} + +static int set_jetty_jfr(struct udma_dev *dev, struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg, uint32_t jfr_id) +{ + if (cfg->jfr) { + jetty->shared_jfr = true; + jetty->udma_jfr = to_udma_jfr(cfg->jfr); + } else { + jetty->shared_jfr = false; + jetty->udma_jfr = get_udma_jfr(&dev->ub_dev, jfr_id); + if (!jetty->udma_jfr) { + dev_err(dev->dev, + "failed to find jfr, jfr_id:%u.\n", jfr_id); + return -EINVAL; + } + } + + return 0; +} + +static int alloc_jetty_um_qp(struct udma_dev *dev, struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg, + struct ubcore_udata *udata) +{ + int ret; + + udma_fill_jetty_um_qp_attr(dev, jetty, udata->uctx, cfg); + + ret = udma_create_qp_common(dev, &jetty->qp, udata); + if (ret) { + dev_err(dev->dev, "failed create qp for um jetty.\n"); + return ret; + } + + jetty->qp.state = QPS_RESET; + ret = udma_modify_qp_jetty(dev, jetty, QPS_RTS); + if (ret) + udma_destroy_qp_common(dev, &jetty->qp); + + return ret; +} + +static void set_jetty_ext_sge_param(struct udma_jetty *jetty) +{ + struct ubcore_jetty_cfg *jetty_cfg = &jetty->ubcore_jetty.jetty_cfg; + uint32_t max_inline_data; + uint32_t wqe_sge_cnt = 0; + uint32_t total_sge_cnt; + uint32_t ext_sge_cnt; + uint32_t sq_wqe_cnt; + uint32_t max_gs; + + sq_wqe_cnt = jetty->rc_node.wqe_cnt; + jetty->rc_node.sge_shift = UDMA_SGE_SHIFT; + max_inline_data = roundup_pow_of_two(jetty_cfg->max_inline_data); + ext_sge_cnt = max_inline_data / UDMA_SGE_SIZE; + + max_gs = max_t(uint32_t, ext_sge_cnt, jetty_cfg->max_send_sge); + if (max_gs > UDMA_SGE_IN_WQE) + wqe_sge_cnt = max_gs - UDMA_SGE_IN_WQE; + + if (wqe_sge_cnt) { + total_sge_cnt = roundup_pow_of_two(sq_wqe_cnt * wqe_sge_cnt); + jetty->rc_node.sge_cnt = max(total_sge_cnt, + (uint32_t)UDMA_PAGE_SIZE / + UDMA_SGE_SIZE); + } +} + +static int set_jetty_buf_attr(struct udma_dev *udma_dev, + struct udma_jetty *jetty, + struct udma_buf_attr *buf_attr) +{ + int totle_buff_size = 0; + uint32_t cfg_depth; + int buf_size; + int idx = 0; + + /* SQ WQE */ + jetty->rc_node.sge_offset = 0; + cfg_depth = roundup_pow_of_two(jetty->ubcore_jetty.jetty_cfg.jfs_depth); + jetty->rc_node.wqe_cnt = cfg_depth < UDMA_MIN_JFS_DEPTH ? + UDMA_MIN_JFS_DEPTH : cfg_depth; + jetty->rc_node.wqe_shift = UDMA_SQ_WQE_SHIFT; + set_jetty_ext_sge_param(jetty); + + buf_size = to_udma_hem_entries_size(jetty->rc_node.wqe_cnt, + jetty->rc_node.wqe_shift); + if (buf_size > 0) { + buf_attr->region[idx].size = buf_size; + buf_attr->region[idx].hopnum = udma_dev->caps.wqe_sq_hop_num; + idx++; + totle_buff_size += buf_size; + } + /* extend SGE WQE in SQ */ + jetty->rc_node.sge_offset = totle_buff_size; + + buf_size = to_udma_hem_entries_size(jetty->rc_node.sge_cnt, + jetty->rc_node.sge_shift); + if (buf_size > 0) { + buf_attr->region[idx].size = buf_size; + buf_attr->region[idx].hopnum = udma_dev->caps.wqe_sge_hop_num; + idx++; + totle_buff_size += buf_size; + } + + if (totle_buff_size < 1) + return -EINVAL; + + buf_attr->region_count = idx; + buf_attr->page_shift = PAGE_SHIFT + udma_dev->caps.mtt_buf_pg_sz; + + return 0; +} + +static int alloc_jetty_buf(struct udma_dev *dev, struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg, struct ubcore_udata *udata) +{ + struct udma_create_jetty_ucmd ucmd = {}; + struct udma_buf_attr buf_attr = {}; + int ret; + + if (udata) { + ret = copy_from_user(&ucmd, (void *)udata->udrv_data->in_addr, + min(udata->udrv_data->in_len, + (uint32_t)sizeof(ucmd))); + if (ret) { + dev_err(dev->dev, + "failed to copy jetty udata, ret = %d.\n", + ret); + return -EFAULT; + } + } + + ret = set_jetty_jfr(dev, jetty, cfg, ucmd.jfr_id); + if (ret) + return ret; + + if (cfg->trans_mode == UBCORE_TP_UM) { + ret = alloc_jetty_um_qp(dev, jetty, cfg, udata); + if (ret) + return ret; + } else if (cfg->trans_mode == UBCORE_TP_RM) { + xa_init(&jetty->srm_node_table); + } else if (cfg->trans_mode == UBCORE_TP_RC) { + jetty->rc_node.buf_addr = ucmd.buf_addr; + ret = set_jetty_buf_attr(dev, jetty, &buf_attr); + if (ret) { + dev_err(dev->dev, + "failed to set jetty buf attr, ret = %d.\n", + ret); + return ret; + } + + ret = udma_mtr_create(dev, &jetty->rc_node.mtr, &buf_attr, + PAGE_SHIFT + dev->caps.mtt_ba_pg_sz, + ucmd.buf_addr, !!udata); + if (ret) { + dev_err(dev->dev, + "failed to create WQE mtr for RC Jetty, ret = %d.\n", + ret); + return ret; + } + + ret = udma_db_map_user(dev, ucmd.sdb_addr, &jetty->rc_node.sdb); + if (ret) { + dev_err(dev->dev, + "failed to map user sdb_addr, ret = %d.\n", + ret); + udma_mtr_destroy(dev, &jetty->rc_node.mtr); + return ret; + } + } + + return 0; +} + +static int alloc_jetty_id(struct udma_dev *udma_dev, struct udma_jetty *jetty) +{ + struct udma_jetty_table *jetty_table = &udma_dev->jetty_table; + struct udma_ida *jetty_ida = &jetty_table->jetty_ida; + int ret; + int id; + + id = ida_alloc_range(&jetty_ida->ida, jetty_ida->min, jetty_ida->max, + GFP_KERNEL); + if (id < 0) { + dev_err(udma_dev->dev, "failed to alloc jetty_id(%d).\n", id); + return id; + } + jetty->jetty_id = (uint32_t)id; + jetty->ubcore_jetty.id = jetty->jetty_id; + + ret = xa_err(xa_store(&jetty_table->xa, jetty->jetty_id, jetty, + GFP_KERNEL)); + if (ret) { + dev_err(udma_dev->dev, "failed to store Jetty, ret = %d.\n", + ret); + ida_free(&jetty_ida->ida, id); + } + + return ret; +} + + +static void free_jetty_id(struct udma_dev *udma_dev, struct udma_jetty *jetty) +{ + struct udma_jetty_table *jetty_table = &udma_dev->jetty_table; + struct udma_ida *jetty_ida = &jetty_table->jetty_ida; + + xa_erase(&jetty_table->xa, jetty->jetty_id); + ida_free(&jetty_ida->ida, (int)jetty->jetty_id); +} + +struct ubcore_jetty *udma_create_jetty(struct ubcore_device *dev, + const struct ubcore_jetty_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + struct udma_jetty *jetty; + int ret; + + jetty = kzalloc(sizeof(struct udma_jetty), GFP_KERNEL); + if (!jetty) + return NULL; + + init_jetty_cfg(jetty, cfg); + ret = alloc_jetty_id(udma_dev, jetty); + if (ret) + goto err_alloc_jetty_id; + + init_jetty_x_qpn_bitmap(udma_dev, &jetty->qpn_map, + udma_dev->caps.num_jetty_shift, + UDMA_JETTY_QPN_PREFIX, jetty->jetty_id); + + ret = alloc_jetty_buf(udma_dev, jetty, cfg, udata); + if (ret) { + dev_err(udma_dev->dev, "alloc Jetty buf failed.\n"); + goto err_alloc_jetty_buf; + } + + mutex_init(&jetty->tp_mutex); + + return &jetty->ubcore_jetty; + +err_alloc_jetty_buf: + clean_jetty_x_qpn_bitmap(&jetty->qpn_map); + free_jetty_id(udma_dev, jetty); +err_alloc_jetty_id: + kfree(jetty); + + return NULL; +} + +int free_jetty_buf(struct udma_dev *dev, struct udma_jetty *jetty) +{ + int ret = 0; + + if (jetty->tp_mode == UBCORE_TP_UM) { + ret = udma_modify_qp_jetty(dev, jetty, QPS_RESET); + if (ret) + dev_err(dev->dev, + "modify qp(0x%llx) to RESET failed for um jetty.\n", + jetty->qp.qpn); + + udma_destroy_qp_common(dev, &jetty->qp); + } else if (jetty->tp_mode == UBCORE_TP_RC) { + udma_db_unmap_user(dev, &jetty->rc_node.sdb); + udma_mtr_destroy(dev, &jetty->rc_node.mtr); + } + + return ret; +} + +int udma_destroy_jetty(struct ubcore_jetty *jetty) +{ + struct udma_jetty *udma_jetty; + struct udma_dev *udma_dev; + int ret; + + udma_jetty = to_udma_jetty(jetty); + udma_dev = to_udma_dev(jetty->ub_dev); + + ret = free_jetty_buf(udma_dev, udma_jetty); + clean_jetty_x_qpn_bitmap(&udma_jetty->qpn_map); + free_jetty_id(udma_dev, udma_jetty); + kfree(udma_jetty); + + return ret; +} diff --git a/drivers/ub/hw/hns3/hns3_udma_jetty.h b/drivers/ub/hw/hns3/hns3_udma_jetty.h new file mode 100644 index 000000000000..9efc58cbafc4 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_jetty.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_JETTY_H +#define _UDMA_JETTY_H + +#include "hns3_udma_qp.h" + +struct rc_node { + struct udma_tp *tp; + uint32_t tpn; + uint64_t buf_addr; + struct udma_mtr mtr; + uint32_t wqe_cnt; + uint32_t wqe_shift; + uint32_t sge_offset; + uint32_t sge_cnt; + uint32_t sge_shift; + struct udma_db sdb; + struct ubcore_jetty_id tjetty_id; +}; + +struct udma_jetty { + struct ubcore_jetty ubcore_jetty; + bool shared_jfr; + struct udma_jfr *udma_jfr; + enum ubcore_transport_mode tp_mode; + union { + struct rc_node rc_node; + struct xarray srm_node_table; + struct udma_qp qp; + }; + struct udma_qpn_bitmap qpn_map; + uint32_t jetty_id; + struct mutex tp_mutex; +}; + +static inline struct udma_jetty *to_udma_jetty(struct ubcore_jetty *ubcore_jetty) +{ + return container_of(ubcore_jetty, struct udma_jetty, ubcore_jetty); +} + +struct ubcore_jetty *udma_create_jetty(struct ubcore_device *dev, + const struct ubcore_jetty_cfg *cfg, + struct ubcore_udata *udata); +int udma_destroy_jetty(struct ubcore_jetty *jetty); + +#endif /* _UDMA_JETTY_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_jfr.c b/drivers/ub/hw/hns3/hns3_udma_jfr.c index a9d4d600b808..e86c553fb367 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jfr.c +++ b/drivers/ub/hw/hns3/hns3_udma_jfr.c @@ -519,6 +519,20 @@ int udma_destroy_jfr(struct ubcore_jfr *jfr) return 0; } +struct udma_jfr *get_udma_jfr(struct ubcore_device *dev, uint32_t jfr_id) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + struct udma_jfr *udma_jfr; + + udma_jfr = (struct udma_jfr *)xa_load(&udma_dev->jfr_table.xa, jfr_id); + if (IS_ERR_OR_NULL(udma_jfr)) { + dev_err(&dev->dev, "failed to find jfr, jfr_id:%u.", jfr_id); + return NULL; + } + + return udma_jfr; +} + struct ubcore_tjetty *udma_import_jfr(struct ubcore_device *dev, const struct ubcore_tjetty_cfg *cfg, struct ubcore_udata *udata) diff --git a/drivers/ub/hw/hns3/hns3_udma_jfr.h b/drivers/ub/hw/hns3/hns3_udma_jfr.h index 838aa46fa759..c804889e3ac1 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jfr.h +++ b/drivers/ub/hw/hns3/hns3_udma_jfr.h @@ -87,6 +87,7 @@ static inline struct udma_jfr *to_udma_jfr(struct ubcore_jfr *ubcore_jfr) struct ubcore_jfr *udma_create_jfr(struct ubcore_device *dev, const struct ubcore_jfr_cfg *cfg, struct ubcore_udata *udata); int udma_destroy_jfr(struct ubcore_jfr *jfr); +struct udma_jfr *get_udma_jfr(struct ubcore_device *dev, uint32_t jfr_id); struct ubcore_tjetty *udma_import_jfr(struct ubcore_device *dev, const struct ubcore_tjetty_cfg *cfg, struct ubcore_udata *udata); diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c index bb78d0f2587a..2622525d893e 100644 --- a/drivers/ub/hw/hns3/hns3_udma_main.c +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -25,6 +25,7 @@ #include "hns3_udma_jfc.h" #include "hns3_udma_jfs.h" #include "hns3_udma_segment.h" +#include "hns3_udma_jetty.h" #include "hns3_udma_cmd.h" static int udma_set_eid(struct ubcore_device *dev, union ubcore_eid eid) @@ -330,6 +331,8 @@ static struct ubcore_ops g_udma_dev_ops = { .destroy_jfr = udma_destroy_jfr, .import_jfr = udma_import_jfr, .unimport_jfr = udma_unimport_jfr, + .create_jetty = udma_create_jetty, + .destroy_jetty = udma_destroy_jetty, .create_tp = udma_create_tp, .modify_tp = udma_modify_tp, .destroy_tp = udma_destroy_tp, diff --git a/drivers/ub/hw/hns3/hns3_udma_qp.c b/drivers/ub/hw/hns3/hns3_udma_qp.c index 427ea1c112f4..2d27f24ab0ef 100644 --- a/drivers/ub/hw/hns3/hns3_udma_qp.c +++ b/drivers/ub/hw/hns3/hns3_udma_qp.c @@ -16,10 +16,14 @@ #include #include #include "hns3_udma_abi.h" +#include "hns3_udma_jfs.h" #include "hns3_udma_jfr.h" #include "hns3_udma_jfc.h" #include "hns3_udma_hem.h" #include "hns3_udma_cmd.h" +#include "hns3_udma_jetty.h" +#include "hns3_udma_tp.h" +#include "hns3_udma_db.h" #include "hns3_udma_qp.h" static void set_qpc_wqe_cnt(struct udma_qp *qp, @@ -616,6 +620,48 @@ int udma_modify_qp_common(struct udma_qp *qp, return ret; } +int fill_jfs_qp_attr(struct udma_dev *udma_dev, struct udma_qp_attr *qp_attr, + struct udma_create_tp_ucmd *ucmd) +{ + struct udma_jfs *udma_jfs; + struct ubcore_jfs *jfs; + struct ubcore_jfc *jfc; + + udma_jfs = (struct udma_jfs *)xa_load(&udma_dev->jfs_table.xa, + ucmd->ini_id.jfs_id); + if (IS_ERR_OR_NULL(udma_jfs)) { + dev_err(udma_dev->dev, "failed to find jfs\n"); + return -EINVAL; + } + jfs = &udma_jfs->ubcore_jfs; + jfc = jfs->jfs_cfg.jfc; + qp_attr->send_jfc = to_udma_jfc(jfc); + qp_attr->recv_jfc = NULL; + qp_attr->jfs = udma_jfs; + qp_attr->qpn_map = &qp_attr->jfs->qpn_map; + qp_attr->uctx = qp_attr->jfs->ubcore_jfs.uctx; + qp_attr->cap.max_send_wr = jfs->jfs_cfg.depth; + qp_attr->cap.max_send_sge = jfs->jfs_cfg.max_sge; + qp_attr->cap.max_inline_data = jfs->jfs_cfg.max_inline_data; + qp_attr->cap.retry_cnt = jfs->jfs_cfg.retry_cnt; + qp_attr->cap.rnr_retry = jfs->jfs_cfg.rnr_retry; + qp_attr->cap.ack_timeout = jfs->jfs_cfg.err_timeout; + qp_attr->qp_type = QPT_RC; + qp_attr->tgt_id = ucmd->tgt_id.jfr_id; + if (jfs->jfs_cfg.priority >= udma_dev->caps.sl_num) { + qp_attr->priority = udma_dev->caps.sl_num > 0 ? + udma_dev->caps.sl_num - 1 : 0; + dev_err(udma_dev->dev, + "The setted priority (%d) cannot larger than the max priority (%d), priority (%d) is used.\n", + jfs->jfs_cfg.priority, udma_dev->caps.sl_num, + qp_attr->priority); + } else { + qp_attr->priority = jfs->jfs_cfg.priority; + } + + return 0; +} + int fill_jfr_qp_attr(struct udma_dev *udma_dev, struct udma_qp_attr *qp_attr, struct udma_create_tp_ucmd *ucmd) { @@ -637,10 +683,72 @@ int fill_jfr_qp_attr(struct udma_dev *udma_dev, struct udma_qp_attr *qp_attr, if (jfr->jfr_cfg.trans_mode == UBCORE_TP_UM) { dev_err(udma_dev->dev, "jfr tp mode error\n"); return -EINVAL; + } else { + qp_attr->qp_type = QPT_RC; + } + qp_attr->cap.min_rnr_timer = jfr->jfr_cfg.min_rnr_timer; + + return 0; +} + +int fill_jetty_qp_attr(struct udma_dev *udma_dev, struct udma_qp_attr *qp_attr, + struct udma_create_tp_ucmd *ucmd) +{ + struct udma_jetty *udma_jetty; + struct ubcore_jetty *jetty; + uint32_t jetty_id; + + jetty_id = qp_attr->is_tgt ? ucmd->tgt_id.jetty_id : + ucmd->ini_id.jetty_id; + qp_attr->tgt_id = qp_attr->is_tgt ? ucmd->ini_id.jetty_id : + ucmd->tgt_id.jetty_id; + + udma_jetty = (struct udma_jetty *)xa_load(&udma_dev->jetty_table.xa, + jetty_id); + if (IS_ERR_OR_NULL(udma_jetty)) { + dev_err(udma_dev->dev, "failed to find jetty\n"); + return -EINVAL; } + jetty = &udma_jetty->ubcore_jetty; + if (udma_jetty->tp_mode == UBCORE_TP_UM) { + dev_err(udma_dev->dev, "jetty tp mode error\n"); + return -EINVAL; + } + + qp_attr->jetty = udma_jetty; + if (!qp_attr->is_tgt || udma_jetty->tp_mode == UBCORE_TP_RC) { + qp_attr->uctx = jetty->uctx; + qp_attr->qpn_map = &udma_jetty->qpn_map; + qp_attr->send_jfc = to_udma_jfc(jetty->jetty_cfg.send_jfc); + qp_attr->cap.max_send_wr = jetty->jetty_cfg.jfs_depth; + qp_attr->cap.max_send_sge = jetty->jetty_cfg.max_send_sge; + qp_attr->cap.max_inline_data = jetty->jetty_cfg.max_inline_data; + if (jetty->jetty_cfg.priority >= udma_dev->caps.sl_num) { + qp_attr->priority = udma_dev->caps.sl_num > 0 ? + udma_dev->caps.sl_num - 1 : 0; + dev_err(udma_dev->dev, + "The setted priority (%d) should smaller than the max priority (%d), priority (%d) is used\n", + jetty->jetty_cfg.priority, + udma_dev->caps.sl_num, qp_attr->priority); + } else { + qp_attr->priority = jetty->jetty_cfg.priority; + } + } + + qp_attr->jfr = udma_jetty->udma_jfr; + qp_attr->uctx = udma_jetty->udma_jfr->ubcore_jfr.uctx; + qp_attr->qpn_map = &udma_jetty->qpn_map; + qp_attr->recv_jfc = + to_udma_jfc(udma_jetty->udma_jfr->ubcore_jfr.jfr_cfg.jfc); + qp_attr->qp_type = QPT_RC; - qp_attr->cap.min_rnr_timer = jfr->jfr_cfg.min_rnr_timer; + qp_attr->cap.min_rnr_timer = + udma_jetty->udma_jfr->ubcore_jfr.jfr_cfg.min_rnr_timer; + + qp_attr->cap.retry_cnt = jetty->jetty_cfg.retry_cnt; + qp_attr->cap.ack_timeout = jetty->jetty_cfg.err_timeout; + qp_attr->cap.rnr_retry = jetty->jetty_cfg.rnr_retry; return 0; } @@ -677,16 +785,62 @@ int udma_fill_qp_attr(struct udma_dev *udma_dev, struct udma_qp_attr *qp_attr, if (!is_target) { qp_attr->pdn = udma_ctx->pdn; - /* TODO: fill_jfs_qp_attr */ - return 0; + if (!ucmd.is_jetty) + return fill_jfs_qp_attr(udma_dev, qp_attr, &ucmd); + else + return fill_jetty_qp_attr(udma_dev, qp_attr, &ucmd); + } else { + if (!ucmd.is_jetty) + return fill_jfr_qp_attr(udma_dev, qp_attr, &ucmd); + else + return fill_jetty_qp_attr(udma_dev, qp_attr, &ucmd); } - if (!ucmd.is_jetty) - return fill_jfr_qp_attr(udma_dev, qp_attr, &ucmd); - return status; } +static uint32_t get_wqe_ext_sge_cnt(struct udma_qp *qp) +{ + /* UD QP only has extended sge */ + if (qp->qp_type == QPT_UD) + return qp->sq.max_gs; + + if (qp->sq.max_gs > UDMA_SGE_IN_WQE) + return qp->sq.max_gs - UDMA_SGE_IN_WQE; + + return 0; +} + +static void set_ext_sge_param(struct udma_dev *udma_dev, uint32_t sq_wqe_cnt, + struct udma_qp *qp, struct udma_qp_cap *cap) +{ + uint32_t max_inline_data; + uint32_t total_sge_cnt; + uint32_t ext_sge_cnt; + uint32_t wqe_sge_cnt; + + qp->sge.sge_shift = UDMA_SGE_SHIFT; + + max_inline_data = roundup_pow_of_two(cap->max_inline_data); + ext_sge_cnt = max_inline_data / UDMA_SGE_SIZE; + + /* Select the max data set by the user */ + qp->sq.max_gs = max(ext_sge_cnt, cap->max_send_sge); + + wqe_sge_cnt = get_wqe_ext_sge_cnt(qp); + /* If the number of extended sge is not zero, they MUST use the + * space of UDMA_EP_PAGE_SIZE at least. + */ + if (wqe_sge_cnt) { + total_sge_cnt = roundup_pow_of_two(sq_wqe_cnt * wqe_sge_cnt); + qp->sge.sge_cnt = max(total_sge_cnt, + (uint32_t)UDMA_PAGE_SIZE / UDMA_SGE_SIZE); + } + + /* Ensure that the max_gs size does not exceed */ + qp->sq.max_gs = min(qp->sq.max_gs, udma_dev->caps.max_sq_sg); +} + static void set_rq_size(struct udma_qp *qp, struct udma_qp_cap *cap) { /* set rq param to 0 */ @@ -696,6 +850,34 @@ static void set_rq_size(struct udma_qp *qp, struct udma_qp_cap *cap) cap->max_recv_sge = 0; } +static int set_user_sq_size(struct udma_dev *udma_dev, struct udma_qp *qp, + struct udma_qp_cap *cap) +{ + uint32_t cfg_depth; + + if (cap->max_send_wr > udma_dev->caps.max_wqes || + cap->max_send_sge > udma_dev->caps.max_sq_sg) + return -EINVAL; + + qp->sq.wqe_shift = UDMA_SQ_WQE_SHIFT; + cfg_depth = roundup_pow_of_two(cap->max_send_wr); + qp->sq.wqe_cnt = cfg_depth < UDMA_MIN_JFS_DEPTH ? + UDMA_MIN_JFS_DEPTH : cfg_depth; + + set_ext_sge_param(udma_dev, qp->sq.wqe_cnt, qp, cap); + + return 0; +} + +static bool is_rc_jetty(struct udma_qp_attr *qp_attr) +{ + if (qp_attr->is_jetty && qp_attr->jetty && + qp_attr->jetty->tp_mode == UBCORE_TP_RC) + return true; + + return false; +} + static int set_qp_param(struct udma_dev *udma_dev, struct udma_qp *qp, struct ubcore_udata *udata, struct udma_create_tp_ucmd *ucmd) @@ -710,7 +892,17 @@ static int set_qp_param(struct udma_dev *udma_dev, struct udma_qp *qp, qp->retry_cnt = qp_attr->cap.retry_cnt; qp->ack_timeout = qp_attr->cap.ack_timeout; qp->rnr_retry = qp_attr->cap.rnr_retry; + if (qp_attr->is_jetty) + qp->min_rnr_timer = qp_attr->cap.min_rnr_timer; qp->priority = qp_attr->priority; + } else { + qp->min_rnr_timer = qp_attr->cap.min_rnr_timer; + if (qp_attr->is_jetty) { + qp->retry_cnt = qp_attr->cap.retry_cnt; + qp->ack_timeout = qp_attr->cap.ack_timeout; + qp->rnr_retry = qp_attr->cap.rnr_retry; + qp->priority = qp_attr->priority; + } } if (qp_attr->cap.max_inline_data > udma_dev->caps.max_sq_inline) @@ -729,6 +921,18 @@ static int set_qp_param(struct udma_dev *udma_dev, struct udma_qp *qp, return ret; } + ret = set_user_sq_size(udma_dev, qp, &qp_attr->cap); + if (ret) + dev_err(dev, + "failed to set user SQ size, ret = %d.\n", ret); + } else { + if (is_rc_jetty(qp_attr)) { + ret = set_user_sq_size(udma_dev, qp, &qp_attr->cap); + if (ret) + dev_err(dev, + "failed to set user SQ size for RC Jetty, ret = %d.\n", + ret); + } } return ret; @@ -855,6 +1059,123 @@ void clean_jetty_x_qpn_bitmap(struct udma_qpn_bitmap *qpn_map) mutex_unlock(&qpn_map->bank_mutex); } +static int set_wqe_buf_attr(struct udma_dev *udma_dev, struct udma_qp *qp, + struct udma_buf_attr *buf_attr) +{ + uint32_t idx = 0; + int buf_size; + + qp->buff_size = 0; + + /* SQ WQE */ + qp->sq.offset = 0; + + buf_size = to_udma_hem_entries_size(qp->sq.wqe_cnt, + qp->sq.wqe_shift); + if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) { + buf_attr->region[idx].size = buf_size; + buf_attr->region[idx].hopnum = udma_dev->caps.wqe_sq_hop_num; + idx++; + qp->buff_size += buf_size; + } + /* extend SGE WQE in SQ */ + qp->sge.offset = qp->buff_size; + + buf_size = to_udma_hem_entries_size(qp->sge.sge_cnt, + qp->sge.sge_shift); + if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) { + buf_attr->region[idx].size = buf_size; + buf_attr->region[idx].hopnum = udma_dev->caps.wqe_sge_hop_num; + idx++; + qp->buff_size += buf_size; + } + + if (qp->buff_size < 1) + return -EINVAL; + + buf_attr->region_count = idx; + buf_attr->page_shift = UDMA_HW_PAGE_SHIFT + udma_dev->caps.mtt_buf_pg_sz; + + return 0; +} + +static int alloc_wqe_buf(struct udma_dev *dev, struct udma_qp *qp, + struct udma_buf_attr *buf_attr, uint64_t addr) +{ + int ret; + + if ((PAGE_SIZE <= UDMA_DWQE_SIZE) && + (dev->caps.flags & UDMA_CAP_FLAG_DIRECT_WQE) && + (qp->qpn < UDMA_DWQE_MMAP_QP_NUM)) + qp->en_flags |= UDMA_QP_CAP_DIRECT_WQE; + + ret = udma_mtr_create(dev, &qp->mtr, buf_attr, + PAGE_SHIFT + dev->caps.mtt_ba_pg_sz, addr, true); + if (ret) + dev_err(dev->dev, "failed to create WQE mtr, ret = %d.\n", ret); + + return ret; +} + +static int alloc_qp_wqe(struct udma_dev *udma_dev, struct udma_qp *qp, + uint64_t buf_addr) +{ + struct device *dev = udma_dev->dev; + struct udma_buf_attr buf_attr = {}; + int ret; + + ret = set_wqe_buf_attr(udma_dev, qp, &buf_attr); + if (ret) { + dev_err(dev, "failed to set WQE attr, ret = %d.\n", ret); + return ret; + } + + ret = alloc_wqe_buf(udma_dev, qp, &buf_attr, buf_addr); + if (ret) { + dev_err(dev, "failed to alloc WQE buf, ret = %d.\n", ret); + return ret; + } + + return 0; +} + +static int alloc_user_qp_db(struct udma_dev *udma_dev, struct udma_qp *qp, + struct udma_create_tp_ucmd *ucmd) +{ + int ret; + + if (!ucmd->sdb_addr) + return 0; + + ret = udma_db_map_user(udma_dev, ucmd->sdb_addr, &qp->sdb); + if (ret) { + dev_err(udma_dev->dev, + "failed to map user sdb_addr, ret = %d.\n", ret); + return ret; + } + + qp->en_flags |= UDMA_QP_CAP_SQ_RECORD_DB; + + return 0; +} + +static int alloc_qp_db(struct udma_dev *udma_dev, struct udma_qp *qp, + struct ubcore_udata *udata, + struct udma_create_tp_ucmd *ucmd) +{ + int ret; + + if (udma_dev->caps.flags & UDMA_CAP_FLAG_SDI_MODE) + qp->en_flags |= UDMA_QP_CAP_OWNER_DB; + + if (udata) { + ret = alloc_user_qp_db(udma_dev, qp, ucmd); + if (ret) + return ret; + } + + return 0; +} static int alloc_qpc(struct udma_dev *udma_dev, struct udma_qp *qp) { @@ -967,6 +1288,14 @@ static void udma_unlock_cqs(struct udma_jfc *send_jfc, } } +void copy_send_jfc(struct udma_qp *from_qp, struct udma_qp *to_qp) +{ + to_qp->qp_attr.send_jfc = from_qp->qp_attr.send_jfc; + udma_lock_cqs(to_qp->qp_attr.send_jfc, NULL); + list_add_tail(&to_qp->sq_node, &to_qp->qp_attr.send_jfc->sq_list); + udma_unlock_cqs(to_qp->qp_attr.send_jfc, NULL); +} + static void add_qp_to_list(struct udma_dev *udma_dev, struct udma_qp *qp, struct udma_jfc *send_jfc, struct udma_jfc *recv_jfc) { @@ -1049,6 +1378,28 @@ static void free_qpc(struct udma_dev *udma_dev, struct udma_qp *qp) udma_table_put(udma_dev, &qp_table->irrl_table, qp->qpn); } +static void free_qp_db(struct udma_dev *udma_dev, struct udma_qp *qp) +{ + if (is_rc_jetty(&qp->qp_attr) || qp->no_free_wqe_buf) + return; + + if (qp->en_flags & UDMA_QP_CAP_SQ_RECORD_DB) + udma_db_unmap_user(udma_dev, &qp->sdb); +} + +static void free_wqe_buf(struct udma_dev *dev, struct udma_qp *qp) +{ + if (is_rc_jetty(&qp->qp_attr) || qp->no_free_wqe_buf) + return; + + udma_mtr_destroy(dev, &qp->mtr); +} + +static void free_qp_wqe(struct udma_dev *udma_dev, struct udma_qp *qp) +{ + free_wqe_buf(udma_dev, qp); +} + static inline uint8_t get_qp_bankid(uint64_t qpn) { /* The lower 3 bits of QPN are used to hash to different banks */ @@ -1075,9 +1426,53 @@ static void free_qpn(struct udma_qp *qp) mutex_unlock(&qpn_map->bank_mutex); } +bool udma_qp_need_alloc_sq(struct udma_qp_attr *qp_attr) +{ + if (qp_attr->is_jetty) { + /* create qp for UM jetty */ + if (qp_attr->jetty->tp_mode == UBCORE_TP_UM) + return true; + + if (qp_attr->jetty->tp_mode == UBCORE_TP_RC) + return true; + + return !qp_attr->is_tgt; + } + + /* create qp for jfs for send */ + if (qp_attr->jfs) + return true; + + /* create qp for jfr for recv */ + return false; +} + +static uint32_t udma_get_jetty_qpn(struct udma_qp *qp) +{ + struct udma_tp *tp = container_of(qp, struct udma_tp, qp); + struct udma_jetty *jetty = qp->qp_attr.jetty; + uint32_t qpn = qp->qpn; + struct udma_tp *pre_tp; + uint32_t hash; + + if (jetty->tp_mode == UBCORE_TP_RC && jetty->rc_node.tp != NULL) + qpn = jetty->rc_node.tpn; + + if (jetty->tp_mode == UBCORE_TP_RM) { + hash = udma_get_jetty_hash(&tp->tjetty_id); + pre_tp = + (struct udma_tp *)xa_load(&jetty->srm_node_table, hash); + if (pre_tp) + qpn = pre_tp->qp.qpn; + } + + return qpn; +} + int udma_create_qp_common(struct udma_dev *udma_dev, struct udma_qp *qp, struct ubcore_udata *udata) { + struct udma_qp_attr *qp_attr = &qp->qp_attr; struct device *dev = udma_dev->dev; struct udma_create_tp_ucmd ucmd; struct udma_create_tp_resp resp; @@ -1102,6 +1497,30 @@ int udma_create_qp_common(struct udma_dev *udma_dev, struct udma_qp *qp, goto err_qpn; } + if (udma_qp_need_alloc_sq(qp_attr)) { + if (is_rc_jetty(qp_attr)) { + qp->mtr = qp_attr->jetty->rc_node.mtr; + qp->sdb = qp_attr->jetty->rc_node.sdb; + qp->en_flags |= UDMA_QP_CAP_SQ_RECORD_DB; + } else { + ret = alloc_qp_wqe(udma_dev, qp, ucmd.buf_addr); + if (ret) { + dev_err(dev, + "failed to alloc QP buffer, ret = %d.\n", + ret); + goto err_buf; + } + + ret = alloc_qp_db(udma_dev, qp, udata, &ucmd); + if (ret) { + dev_err(dev, + "failed to alloc QP doorbell, ret = %d.\n", + ret); + goto err_db; + } + } + } + ret = alloc_qpc(udma_dev, qp); if (ret) { dev_err(dev, "failed to alloc QP context, ret = %d.\n", @@ -1119,6 +1538,9 @@ int udma_create_qp_common(struct udma_dev *udma_dev, struct udma_qp *qp, resp.cap_flags = qp->en_flags; resp.qpn = qp->qpn; resp.priority = qp->priority; + if (qp_attr->is_jetty && qp_attr->jetty) + resp.qpn = udma_get_jetty_qpn(qp); + resp.path_mtu = udma_dev->caps.max_mtu; ret = copy_to_user((void *)udata->udrv_data->out_addr, &resp, min(udata->udrv_data->out_len, @@ -1139,6 +1561,12 @@ int udma_create_qp_common(struct udma_dev *udma_dev, struct udma_qp *qp, err_store: free_qpc(udma_dev, qp); err_qpc: + if (udma_qp_need_alloc_sq(&qp->qp_attr)) + free_qp_db(udma_dev, qp); +err_db: + if (udma_qp_need_alloc_sq(&qp->qp_attr)) + free_qp_wqe(udma_dev, qp); +err_buf: free_qpn(qp); err_qpn: return ret; @@ -1153,6 +1581,10 @@ void udma_destroy_qp_common(struct udma_dev *udma_dev, struct udma_qp *qp) wait_for_completion(&qp->free); free_qpc(udma_dev, qp); + if (udma_qp_need_alloc_sq(&qp->qp_attr) || qp->force_free_wqe_buf) { + free_qp_db(udma_dev, qp); + free_qp_wqe(udma_dev, qp); + } free_qpn(qp); } diff --git a/drivers/ub/hw/hns3/hns3_udma_qp.h b/drivers/ub/hw/hns3/hns3_udma_qp.h index 8e7bd5f8364c..ea5d984ab435 100644 --- a/drivers/ub/hw/hns3/hns3_udma_qp.h +++ b/drivers/ub/hw/hns3/hns3_udma_qp.h @@ -202,6 +202,7 @@ struct udma_qp_attr { struct udma_jfc *recv_jfc; struct udma_jfs *jfs; struct udma_jfr *jfr; + struct udma_jetty *jetty; struct udma_qp_cap cap; enum udma_qp_type qp_type; uint32_t pdn; @@ -211,6 +212,7 @@ struct udma_qp_attr { dma_addr_t reorder_cq_addr; union ubcore_eid remote_eid; union ubcore_eid local_eid; + int tgt_id; uint8_t priority; }; @@ -236,10 +238,12 @@ struct udma_qp { struct udma_qp_attr qp_attr; struct udma_wq sq; struct udma_wq rq; + struct udma_db sdb; struct udma_jfc *send_jfc; struct udma_jfc *recv_jfc; uint64_t en_flags; struct udma_mtr mtr; + uint32_t buff_size; enum udma_qp_state state; void (*event)(struct udma_qp *qp, enum udma_event event_type); @@ -260,6 +264,8 @@ struct udma_qp { uint8_t ack_timeout; uint8_t min_rnr_timer; uint8_t priority; + bool no_free_wqe_buf; + bool force_free_wqe_buf; }; #define gen_qpn(high, mid, low) ((high) | (mid) | (low)) @@ -279,5 +285,6 @@ void init_jetty_x_qpn_bitmap(struct udma_dev *dev, uint32_t jid); void clean_jetty_x_qpn_bitmap(struct udma_qpn_bitmap *qpn_map); void udma_qp_event(struct udma_dev *udma_dev, uint32_t qpn, int event_type); +void copy_send_jfc(struct udma_qp *from_qp, struct udma_qp *to_qp); #endif /* _UDMA_QP_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_segment.c b/drivers/ub/hw/hns3/hns3_udma_segment.c index acb9c2ee9715..159c5a2bd99b 100644 --- a/drivers/ub/hw/hns3/hns3_udma_segment.c +++ b/drivers/ub/hw/hns3/hns3_udma_segment.c @@ -98,7 +98,6 @@ static int alloc_seg_pbl(struct udma_dev *udma_dev, struct udma_seg *seg, buf_attr.region[0].size = seg->size; buf_attr.region[0].hopnum = seg->pbl_hop_num; buf_attr.region_count = 1; - buf_attr.mtt_only = false; err = udma_mtr_create(udma_dev, &seg->pbl_mtr, &buf_attr, udma_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT, diff --git a/drivers/ub/hw/hns3/hns3_udma_tp.c b/drivers/ub/hw/hns3/hns3_udma_tp.c index fa9eb5e06478..2372db943dc2 100644 --- a/drivers/ub/hw/hns3/hns3_udma_tp.c +++ b/drivers/ub/hw/hns3/hns3_udma_tp.c @@ -17,6 +17,8 @@ #include #include "hns3_udma_jfr.h" #include "hns3_udma_jfs.h" +#include "hns3_udma_jetty.h" +#include "hns3_udma_hem.h" #include "hns3_udma_tp.h" static enum udma_qp_state to_udma_qp_state(enum ubcore_tp_state state) @@ -61,8 +63,16 @@ struct udma_modify_tp_attr *udma_get_m_attr(struct ubcore_tp *tp, struct udma_qp m_attr->ack_timeout = qp->ack_timeout; m_attr->rnr_retry = qp->rnr_retry; m_attr->priority = qp->priority; + if (qp->qp_attr.is_jetty) + m_attr->min_rnr_timer = qp->min_rnr_timer; } else { m_attr->min_rnr_timer = qp->min_rnr_timer; + if (qp->qp_attr.is_jetty) { + m_attr->retry_cnt = qp->retry_cnt; + m_attr->ack_timeout = qp->ack_timeout; + m_attr->rnr_retry = qp->rnr_retry; + m_attr->priority = qp->priority; + } } if (mask.bs.peer_tpn) @@ -122,20 +132,44 @@ int udma_modify_tp(struct ubcore_tp *tp, const struct ubcore_tp_attr *attr, void *udma_erase_tp(struct udma_tp *udma_tp) { struct udma_qp_attr *qp_attr; + struct udma_jetty *jetty; + struct udma_tp *pre_tp; struct udma_jfr *jfr; struct udma_jfs *jfs; + uint32_t hash; qp_attr = &udma_tp->qp.qp_attr; + jetty = qp_attr->jetty; jfr = qp_attr->jfr; jfs = qp_attr->jfs; - if (jfr) - return xa_erase(&jfr->tp_table_xa, - udma_tp->ubcore_tp.tpn); - - if (jfs) - return xa_erase(&jfs->node_table, - udma_tp->ubcore_tp.tpn); + if (qp_attr->is_jetty) { + if (jetty->tp_mode == UBCORE_TP_RM) { + hash = udma_get_jetty_hash(&udma_tp->tjetty_id); + pre_tp = (struct udma_tp *)xa_load(&jetty->srm_node_table, + hash); + if (!pre_tp) + return NULL; + + if (pre_tp->qp.qpn == udma_tp->qp.qpn) + xa_erase(&jetty->srm_node_table, hash); + } else if (jetty->tp_mode == UBCORE_TP_RC) { + if (jetty->rc_node.tp == NULL) + return NULL; + + if (jetty->rc_node.tpn == udma_tp->ubcore_tp.tpn) { + jetty->rc_node.tp = NULL; + jetty->rc_node.tpn = 0; + } + } + } else { + if (jfr) + return xa_erase(&jfr->tp_table_xa, + udma_tp->ubcore_tp.tpn); + else if (jfs) + return xa_erase(&jfs->node_table, + udma_tp->ubcore_tp.tpn); + } return udma_tp; } @@ -203,10 +237,80 @@ static void udma_set_tp(struct ubcore_device *dev, const struct ubcore_tp_cfg *c tp->ubcore_tp.state = UBCORE_TP_STATE_RESET; } +static void copy_attr_to_pre_tp(struct udma_qp *from_qp, struct udma_qp *to_qp) +{ + if (from_qp->qp_attr.is_tgt) + return; + + to_qp->sge = from_qp->sge; + to_qp->sq = from_qp->sq; + to_qp->sdb = from_qp->sdb; + to_qp->priority = from_qp->priority; + to_qp->en_flags = from_qp->en_flags; + to_qp->buff_size = from_qp->buff_size; + + udma_mtr_move(&from_qp->mtr, &to_qp->mtr); + copy_send_jfc(from_qp, to_qp); + + to_qp->qp_attr.cap.max_send_wr = from_qp->qp_attr.cap.max_send_wr; + to_qp->qp_attr.cap.max_send_sge = from_qp->qp_attr.cap.max_send_sge; + to_qp->qp_attr.cap.max_inline_data = + from_qp->qp_attr.cap.max_inline_data; + + from_qp->no_free_wqe_buf = true; + to_qp->force_free_wqe_buf = true; +} + +static int udma_store_jetty_tp(struct udma_dev *udma_device, + struct udma_jetty *jetty, struct udma_tp *tp, + struct ubcore_tp **fail_ret_tp) +{ + struct udma_tp *pre_tp; + uint32_t tjetty_hash; + uint32_t hash; + int ret = 0; + + hash = udma_get_jetty_hash(&tp->tjetty_id); + if (jetty->tp_mode == UBCORE_TP_RM) { + pre_tp = (struct udma_tp *)xa_load(&jetty->srm_node_table, hash); + if (pre_tp) { + copy_attr_to_pre_tp(&tp->qp, &pre_tp->qp); + *fail_ret_tp = &pre_tp->ubcore_tp; + return 0; + } + + ret = xa_err(xa_store(&jetty->srm_node_table, + hash, tp, GFP_KERNEL)); + if (ret) + dev_err(udma_device->dev, + "failed store jetty tp xarray, ret = %d\n", ret); + } else if (jetty->tp_mode == UBCORE_TP_RC) { + if (jetty->rc_node.tp == NULL) { + jetty->rc_node.tp = tp; + jetty->rc_node.tpn = tp->ubcore_tp.tpn; + jetty->rc_node.tjetty_id = tp->tjetty_id; + } else { + tjetty_hash = + udma_get_jetty_hash(&jetty->rc_node.tjetty_id); + if (tjetty_hash == hash) { + *fail_ret_tp = &jetty->rc_node.tp->ubcore_tp; + } else { + dev_err(udma_device->dev, + "jetty has bind a target jetty, jetty_id = %d.\n", + jetty->rc_node.tjetty_id.id); + return -EEXIST; + } + } + } + + return ret; +} + static int udma_store_tp(struct udma_dev *udma_device, struct udma_tp *tp, struct ubcore_tp **fail_ret_tp) { struct udma_qp_attr *qp_attr; + struct udma_jetty *jetty; struct udma_jfr *jfr; struct udma_jfs *jfs; int ret = 0; @@ -214,27 +318,56 @@ static int udma_store_tp(struct udma_dev *udma_device, struct udma_tp *tp, qp_attr = &tp->qp.qp_attr; jfr = qp_attr->jfr; jfs = qp_attr->jfs; - if (jfr) { - ret = xa_err(xa_store(&jfr->tp_table_xa, tp->ubcore_tp.tpn, tp, - GFP_KERNEL)); + jetty = qp_attr->jetty; + + if (qp_attr->is_jetty) { + ret = udma_store_jetty_tp(udma_device, jetty, tp, fail_ret_tp); if (ret) { dev_err(udma_device->dev, - "failed store jfr tp, ret = %d\n", ret); + "failed store jetty tp, ret = %d\n", ret); return ret; } - } else if (jfs) { - ret = xa_err(xa_store(&jfs->node_table, tp->ubcore_tp.tpn, tp, - GFP_KERNEL)); - if (ret) { - dev_err(udma_device->dev, - "failed store jfs tp, ret = %d\n", ret); - return ret; + } else { + if (jfr) { + ret = xa_err(xa_store(&jfr->tp_table_xa, + tp->ubcore_tp.tpn, tp, + GFP_KERNEL)); + if (ret) { + dev_err(udma_device->dev, + "failed store jfr tp, ret = %d\n", ret); + return ret; + } + } else if (jfs) { + ret = xa_err(xa_store(&jfs->node_table, + tp->ubcore_tp.tpn, tp, + GFP_KERNEL)); + if (ret) { + dev_err(udma_device->dev, + "failed store jfs tp, ret = %d\n", ret); + return ret; + } } } return ret; } +static void lock_jetty(struct udma_qp_attr *qp_attr) +{ + struct udma_jetty *jetty = qp_attr->jetty; + + if (qp_attr->is_jetty) + mutex_lock(&jetty->tp_mutex); +} + +static void unlock_jetty(struct udma_qp_attr *qp_attr) +{ + struct udma_jetty *jetty = qp_attr->jetty; + + if (qp_attr->is_jetty) + mutex_unlock(&jetty->tp_mutex); +} + struct ubcore_tp *udma_create_tp(struct ubcore_device *dev, const struct ubcore_tp_cfg *cfg, struct ubcore_udata *udata) { @@ -252,17 +385,22 @@ struct ubcore_tp *udma_create_tp(struct ubcore_device *dev, const struct ubcore_ dev_err(udma_dev->dev, "failed to fill qp attr.\n"); goto failed_alloc_tp; } + tp->tjetty_id.id = tp->qp.qp_attr.tgt_id; + tp->tjetty_id.eid = cfg->peer_eid; + lock_jetty(&tp->qp.qp_attr); ret = udma_create_qp_common(udma_dev, &tp->qp, udata); if (ret) { dev_err(udma_dev->dev, "Failed to create qp common with ret is %d.\n", ret); + unlock_jetty(&tp->qp.qp_attr); goto failed_alloc_tp; } udma_set_tp(dev, cfg, tp); ret = udma_store_tp(udma_dev, tp, &fail_ret_tp); + unlock_jetty(&tp->qp.qp_attr); if (ret || fail_ret_tp) goto failed_create_qp; diff --git a/drivers/ub/hw/hns3/hns3_udma_tp.h b/drivers/ub/hw/hns3/hns3_udma_tp.h index 90fa1f27b881..673f2b7fe600 100644 --- a/drivers/ub/hw/hns3/hns3_udma_tp.h +++ b/drivers/ub/hw/hns3/hns3_udma_tp.h @@ -32,6 +32,11 @@ static inline struct udma_tp *to_udma_tp(struct ubcore_tp *ubcore_tp) return container_of(ubcore_tp, struct udma_tp, ubcore_tp); } +static inline uint32_t udma_get_jetty_hash(const struct ubcore_jetty_id *jetty_id) +{ + return jhash(jetty_id, sizeof(struct ubcore_jetty_id), 0); +} + struct ubcore_tp *udma_create_tp(struct ubcore_device *dev, const struct ubcore_tp_cfg *cfg, struct ubcore_udata *udata); -- Gitee From ba75bc59426396a00ee2963c11d0e6c8fa0ce1c0 Mon Sep 17 00:00:00 2001 From: Weibo Zhao Date: Fri, 1 Sep 2023 11:17:33 +0800 Subject: [PATCH 32/42] hns3 udma: add feature of hardware reset driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA ------------------------------------------------------- This patch add feature of hardware reset. In some extreme scenarios, the hardware needs to be reset to try to fix. Driver should exit and reinit after hardware reset. Signed-off-by: Weibo Zhao --- drivers/ub/hw/hns3/hns3_udma_abi.h | 1 + drivers/ub/hw/hns3/hns3_udma_cmd.c | 6 + drivers/ub/hw/hns3/hns3_udma_device.h | 13 ++ drivers/ub/hw/hns3/hns3_udma_hw.c | 259 +++++++++++++++++++++++++- drivers/ub/hw/hns3/hns3_udma_hw.h | 5 + drivers/ub/hw/hns3/hns3_udma_main.c | 11 ++ 6 files changed, 294 insertions(+), 1 deletion(-) diff --git a/drivers/ub/hw/hns3/hns3_udma_abi.h b/drivers/ub/hw/hns3/hns3_udma_abi.h index 9c6628fba99b..4345bf26e4f0 100644 --- a/drivers/ub/hw/hns3/hns3_udma_abi.h +++ b/drivers/ub/hw/hns3/hns3_udma_abi.h @@ -32,6 +32,7 @@ enum { UDMA_MMAP_UAR_PAGE, UDMA_MMAP_DWQE_PAGE, + UDMA_MMAP_RESET_PAGE, }; enum udma_jfc_init_attr_mask { diff --git a/drivers/ub/hw/hns3/hns3_udma_cmd.c b/drivers/ub/hw/hns3/hns3_udma_cmd.c index 452ae6371f3f..7271ab5cc002 100644 --- a/drivers/ub/hw/hns3/hns3_udma_cmd.c +++ b/drivers/ub/hw/hns3/hns3_udma_cmd.c @@ -591,6 +591,12 @@ static int udma_cmd_mbox_wait(struct udma_dev *dev, struct udma_cmq_desc *desc, int udma_cmd_mbox(struct udma_dev *dev, struct udma_cmq_desc *desc, uint32_t timeout, int vfid) { + bool is_busy; + + if (dev->hw->chk_mbox_avail) + if (!dev->hw->chk_mbox_avail(dev, &is_busy)) + return is_busy ? -EBUSY : 0; + if (dev->cmd.use_events) return udma_cmd_mbox_wait(dev, desc, timeout, vfid); else diff --git a/drivers/ub/hw/hns3/hns3_udma_device.h b/drivers/ub/hw/hns3/hns3_udma_device.h index e035dadc5965..b7ef89cb9392 100644 --- a/drivers/ub/hw/hns3/hns3_udma_device.h +++ b/drivers/ub/hw/hns3/hns3_udma_device.h @@ -168,6 +168,12 @@ enum udma_instance_state { UDMA_STATE_UNINIT, }; +#define UDMA_IS_RESETTING 1 + +enum { + UDMA_RST_DIRECT_RETURN = 0, +}; + enum udma_event { UDMA_EVENT_TYPE_COMM_EST = 0x03, UDMA_EVENT_TYPE_WQ_CATAS_ERROR = 0x05, @@ -454,6 +460,10 @@ enum udma_device_state { UDMA_DEVICE_STATE_UNINIT, }; +struct udma_reset_state { + uint32_t reset_state; /* stored to use in user space */ +}; + struct udma_cmq { struct udma_cmq_ring csq; uint16_t tx_timeout; @@ -491,6 +501,7 @@ struct udma_hw { uint16_t token, int event); int (*poll_mbox_done)(struct udma_dev *udma_dev, uint32_t timeout); + bool (*chk_mbox_avail)(struct udma_dev *udma_dev, bool *is_busy); int (*set_hem)(struct udma_dev *udma_dev, struct udma_hem_table *table, int obj, int step_idx); int (*clear_hem)(struct udma_dev *udma_dev, @@ -754,6 +765,8 @@ struct udma_dev { uint64_t sys_image_guid; struct udma_cmdq cmd; int cmd_mod; + struct page *reset_page; /* store reset state */ + void *reset_kaddr; /* addr of reset page */ const struct udma_hw *hw; void *priv; struct workqueue_struct *irq_workq; diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.c b/drivers/ub/hw/hns3/hns3_udma_hw.c index 3fb4b056a650..24fdcf5deb2b 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hw.c +++ b/drivers/ub/hw/hns3/hns3_udma_hw.c @@ -1110,27 +1110,136 @@ static int udma_hw_set_eid(struct udma_dev *udma_dev, union ubcore_eid eid) return ret; } +static void func_clr_hw_resetting_state(struct udma_dev *udma_dev, + struct hnae3_handle *handle) +{ + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + uint64_t end; + + udma_dev->dis_db = true; + + dev_warn(udma_dev->dev, + "Func clear is pending, device in resetting state.\n"); + end = UDMA_HW_RST_TIMEOUT; + while (end) { + if (!ops->get_hw_reset_stat(handle)) { + udma_dev->is_reset = true; + dev_info(udma_dev->dev, + "Func clear success after reset.\n"); + return; + } + msleep(UDMA_HW_RST_COMPLETION_WAIT); + end -= UDMA_HW_RST_COMPLETION_WAIT; + } + + dev_warn(udma_dev->dev, "Func clear failed.\n"); +} + +static void func_clr_sw_resetting_state(struct udma_dev *udma_dev, + struct hnae3_handle *handle) +{ + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + uint64_t end; + + udma_dev->dis_db = true; + + dev_warn(udma_dev->dev, + "Func clear is pending, device in resetting state.\n"); + end = UDMA_HW_RST_TIMEOUT; + while (end) { + if (ops->ae_dev_reset_cnt(handle) != + udma_dev->reset_cnt) { + udma_dev->is_reset = true; + dev_info(udma_dev->dev, + "Func clear success after sw reset\n"); + return; + } + msleep(UDMA_HW_RST_COMPLETION_WAIT); + end -= UDMA_HW_RST_COMPLETION_WAIT; + } + + dev_warn(udma_dev->dev, + "Func clear failed because of unfinished sw reset\n"); +} + +static void udma_func_clr_rst_proc(struct udma_dev *udma_dev, int retval, + int flag) +{ + struct udma_priv *priv = (struct udma_priv *)udma_dev->priv; + struct hnae3_handle *handle = priv->handle; + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + + if (ops->ae_dev_reset_cnt(handle) != udma_dev->reset_cnt) { + udma_dev->dis_db = true; + udma_dev->is_reset = true; + dev_info(udma_dev->dev, "Func clear success after reset.\n"); + return; + } + + if (ops->get_hw_reset_stat(handle)) { + func_clr_hw_resetting_state(udma_dev, handle); + return; + } + + if (ops->ae_dev_resetting(handle) && + handle->udmainfo.instance_state == UDMA_STATE_INIT) { + func_clr_sw_resetting_state(udma_dev, handle); + return; + } + + if (retval && !flag) + dev_warn(udma_dev->dev, + "Func clear read failed, ret = %d.\n", retval); + + dev_warn(udma_dev->dev, "Func clear failed.\n"); +} + +static bool check_device_is_in_reset(struct udma_dev *udma_dev) +{ + struct udma_priv *priv = (struct udma_priv *)udma_dev->priv; + struct hnae3_handle *handle = priv->handle; + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + + if (udma_dev->reset_cnt != ops->ae_dev_reset_cnt(handle)) + return true; + + if (ops->get_hw_reset_stat(handle)) + return true; + + if (ops->ae_dev_resetting(handle)) + return true; + + return false; +} + static void __udma_function_clear(struct udma_dev *udma_dev, int vf_id) { + bool fclr_write_fail_flag = false; struct udma_func_clear *resp; struct udma_cmq_desc desc; uint64_t end; int ret = 0; + if (check_device_is_in_reset(udma_dev)) + goto out; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_FUNC_CLEAR, false); resp = (struct udma_func_clear *)desc.data; resp->rst_funcid_en = cpu_to_le32(vf_id); ret = udma_cmq_send(udma_dev, &desc, 1); if (ret) { + fclr_write_fail_flag = true; dev_err(udma_dev->dev, "Func clear write failed, ret = %d.\n", ret); - return; + goto out; } msleep(UDMA_READ_FUNC_CLEAR_FLAG_INTERVAL); end = UDMA_FUNC_CLEAR_TIMEOUT_MSECS; while (end) { + if (check_device_is_in_reset(udma_dev)) + goto out; msleep(UDMA_READ_FUNC_CLEAR_FLAG_FAIL_WAIT); end -= UDMA_READ_FUNC_CLEAR_FLAG_FAIL_WAIT; @@ -1148,6 +1257,9 @@ static void __udma_function_clear(struct udma_dev *udma_dev, int vf_id) return; } } + +out: + udma_func_clr_rst_proc(udma_dev, ret, fclr_write_fail_flag); } static void udma_free_vf_resource(struct udma_dev *udma_dev, int vf_id) @@ -1311,6 +1423,31 @@ static void udma_free_link_table(struct udma_dev *udma_dev) free_link_table_buf(udma_dev, &priv->ext_llm); } +static int udma_get_reset_page(struct udma_dev *dev) +{ + dev->reset_page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!dev->reset_page) + return -ENOMEM; + + dev->reset_kaddr = vmap(&dev->reset_page, 1, VM_MAP, PAGE_KERNEL); + if (!dev->reset_kaddr) + goto err_vmap; + + return 0; + +err_vmap: + put_page(dev->reset_page); + return -ENOMEM; +} + +static void udma_put_reset_page(struct udma_dev *dev) +{ + vunmap(dev->reset_kaddr); + dev->reset_kaddr = NULL; + put_page(dev->reset_page); + dev->reset_page = NULL; +} + static int udma_clear_extdb_list_info(struct udma_dev *udma_dev) { struct udma_cmq_desc desc; @@ -1401,6 +1538,13 @@ static int udma_hw_init(struct udma_dev *udma_dev) { int ret; + ret = udma_get_reset_page(udma_dev); + if (ret) { + dev_err(udma_dev->dev, + "get reset page failed, ret = %d.\n", ret); + return ret; + } + /* UDMA requires the extdb info to be cleared before using */ ret = udma_clear_extdb_list_info(udma_dev); if (ret) @@ -1421,6 +1565,7 @@ static int udma_hw_init(struct udma_dev *udma_dev) err_llm_init_failed: put_hem_table(udma_dev); err_clear_extdb_failed: + udma_put_reset_page(udma_dev); return ret; } @@ -1627,6 +1772,7 @@ static const struct udma_hw udma_hw = { .hw_exit = udma_hw_exit, .post_mbox = udma_post_mbox, .poll_mbox_done = udma_poll_mbox_done, + .chk_mbox_avail = udma_chk_mbox_is_avail, .set_hem = udma_set_hem, .clear_hem = udma_clear_hem, .set_eid = udma_hw_set_eid, @@ -1729,6 +1875,7 @@ static void __udma_uninit_instance(struct hnae3_handle *handle, static int udma_init_instance(struct hnae3_handle *handle) { + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; struct device *dev = &handle->pdev->dev; const struct pci_device_id *id; int ret; @@ -1746,11 +1893,21 @@ static int udma_init_instance(struct hnae3_handle *handle) return ret; dev_err(dev, "UDMA instance init failed! ret = %d\n", ret); + if (ops->ae_dev_resetting(handle) || + ops->get_hw_reset_stat(handle)) + goto reset_chk_err; + else + return ret; } handle->udmainfo.instance_state = UDMA_STATE_INITED; return 0; + +reset_chk_err: + dev_err(dev, "Device is being reset, please retry later.\n"); + + return -EBUSY; } static void udma_uninit_instance(struct hnae3_handle *handle, bool reset) @@ -1765,6 +1922,105 @@ static void udma_uninit_instance(struct hnae3_handle *handle, bool reset) handle->udmainfo.instance_state = UDMA_STATE_NON_INIT; } +static void udma_reset_notify_user(struct udma_dev *dev) +{ + struct udma_reset_state *state; + + state = (struct udma_reset_state *)dev->reset_kaddr; + + state->reset_state = UDMA_IS_RESETTING; + /* Ensure reset state was flushed in memory */ + wmb(); +} + +static int udma_reset_notify_down(struct hnae3_handle *handle) +{ + struct udma_dev *dev; + + if (handle->udmainfo.instance_state != UDMA_STATE_INITED) { + set_bit(UDMA_RST_DIRECT_RETURN, &handle->udmainfo.state); + return 0; + } + + handle->udmainfo.reset_state = UDMA_STATE_RST_DOWN; + clear_bit(UDMA_RST_DIRECT_RETURN, &handle->udmainfo.state); + + dev = handle->priv; + if (!dev) + return 0; + + dev->dis_db = true; + + udma_reset_notify_user(dev); + + return 0; +} + +static int udma_reset_notify_init(struct hnae3_handle *handle) +{ + struct device *dev = &handle->pdev->dev; + int ret; + + if (test_and_clear_bit(UDMA_RST_DIRECT_RETURN, + &handle->udmainfo.state)) { + handle->udmainfo.reset_state = UDMA_STATE_RST_INITED; + return 0; + } + + handle->udmainfo.reset_state = UDMA_STATE_RST_INIT; + + dev_info(&handle->pdev->dev, "In reset process UDMA client reinit.\n"); + ret = __udma_init_instance(handle); + if (ret) { + /* when reset notify type is HNAE3_INIT_CLIENT In reset notify + * callback function, UB Engine reinitialize. If UDMA reinit + * failed, we should inform NIC driver. + */ + handle->priv = NULL; + dev_err(dev, "In reset process UDMA reinit failed %d.\n", ret); + } else { + handle->udmainfo.reset_state = UDMA_STATE_RST_INITED; + dev_info(dev, "Reset done, UDMA client reinit finished.\n"); + } + + return ret; +} + +static int udma_reset_notify_uninit(struct hnae3_handle *handle) +{ + if (test_bit(UDMA_RST_DIRECT_RETURN, &handle->udmainfo.state)) + return 0; + + handle->udmainfo.reset_state = UDMA_STATE_RST_UNINIT; + dev_info(&handle->pdev->dev, "In reset process UDMA client uninit.\n"); + msleep(UDMA_HW_RST_UNINT_DELAY); + __udma_uninit_instance(handle, false); + + return 0; +} + +static int udma_reset_notify(struct hnae3_handle *handle, + enum hnae3_reset_notify_type type) +{ + int ret = 0; + + switch (type) { + case HNAE3_DOWN_CLIENT: + ret = udma_reset_notify_down(handle); + break; + case HNAE3_INIT_CLIENT: + ret = udma_reset_notify_init(handle); + break; + case HNAE3_UNINIT_CLIENT: + ret = udma_reset_notify_uninit(handle); + break; + default: + break; + } + + return ret; +} + static void udma_link_status_change(struct hnae3_handle *handle, bool linkup) { struct net_device *net_dev; @@ -1809,6 +2065,7 @@ static const struct hnae3_client_ops udma_ops = { .init_instance = udma_init_instance, .uninit_instance = udma_uninit_instance, .link_status_change = udma_link_status_change, + .reset_notify = udma_reset_notify, }; static struct hnae3_client udma_client = { diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.h b/drivers/ub/hw/hns3/hns3_udma_hw.h index 18f8a284cd21..5bc0e119bd64 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hw.h +++ b/drivers/ub/hw/hns3/hns3_udma_hw.h @@ -115,6 +115,11 @@ struct udma_query_oor_cmq { #define UDMA_READ_FUNC_CLEAR_FLAG_INTERVAL 40 #define UDMA_READ_FUNC_CLEAR_FLAG_FAIL_WAIT 20 +#define UDMA_HW_RST_TIMEOUT 1000 +#define UDMA_HW_RST_UNINT_DELAY 100 + +#define UDMA_HW_RST_COMPLETION_WAIT 20 + #define UDMA_FUNC_IRQ_RSV 2 #define UDMA_1US_CFG 999 #define UDMA_EXT_LLM_ENTRY_SZ 8 diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c index 2622525d893e..86104e17c76f 100644 --- a/drivers/ub/hw/hns3/hns3_udma_main.c +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -186,6 +186,15 @@ static int udma_mmap(struct ubcore_ucontext *uctx, struct vm_area_struct *vma) UDMA_DWQE_PAGE_SIZE, vma->vm_page_prot)) return -EAGAIN; break; + case UDMA_MMAP_RESET_PAGE: + if (vma->vm_flags & (VM_WRITE | VM_EXEC)) + return -EINVAL; + + if (remap_pfn_range(vma, vma->vm_start, + page_to_pfn(udma_dev->reset_page), + PAGE_SIZE, vma->vm_page_prot)) + return -EAGAIN; + break; default: dev_err(udma_dev->dev, "mmap failed, cmd(%d) not support\n", cmd); @@ -808,6 +817,8 @@ int udma_hnae_client_init(struct udma_dev *udma_dev) struct device *dev = udma_dev->dev; int ret; + udma_dev->is_reset = false; + ret = udma_dev->hw->cmq_init(udma_dev); if (ret) { dev_err(dev, "Init UB Command Queue failed!\n"); -- Gitee From e44392a8c5242247b82ce50dab3577cbb2b14240 Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Mon, 4 Sep 2023 21:21:02 +0800 Subject: [PATCH 33/42] hns3 udma: support import and unimport jetty driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA --------------------------------------------------------------------- In the kernel mode import_jetty, UDMA driver only needs to alloc memory of the target jetty and return its pointer to the URMA. In addition, unimport_jetty just free the memory of target jetty. Signed-off-by: Yu Zhang --- drivers/ub/hw/hns3/hns3_udma_jetty.c | 20 ++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_jetty.h | 4 ++++ drivers/ub/hw/hns3/hns3_udma_main.c | 2 ++ 3 files changed, 26 insertions(+) diff --git a/drivers/ub/hw/hns3/hns3_udma_jetty.c b/drivers/ub/hw/hns3/hns3_udma_jetty.c index c9cdb22d328e..1200c93d65b3 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jetty.c +++ b/drivers/ub/hw/hns3/hns3_udma_jetty.c @@ -376,3 +376,23 @@ int udma_destroy_jetty(struct ubcore_jetty *jetty) return ret; } + +struct ubcore_tjetty *udma_import_jetty(struct ubcore_device *dev, + const struct ubcore_tjetty_cfg *cfg, + struct ubcore_udata *udata) +{ + struct ubcore_tjetty *tjetty; + + tjetty = kcalloc(1, sizeof(struct ubcore_tjetty), GFP_KERNEL); + if (!tjetty) + return NULL; + + return tjetty; +} + +int udma_unimport_jetty(struct ubcore_tjetty *tjetty) +{ + kfree(tjetty); + + return 0; +} diff --git a/drivers/ub/hw/hns3/hns3_udma_jetty.h b/drivers/ub/hw/hns3/hns3_udma_jetty.h index 9efc58cbafc4..1fe237fbc57c 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jetty.h +++ b/drivers/ub/hw/hns3/hns3_udma_jetty.h @@ -56,5 +56,9 @@ struct ubcore_jetty *udma_create_jetty(struct ubcore_device *dev, const struct ubcore_jetty_cfg *cfg, struct ubcore_udata *udata); int udma_destroy_jetty(struct ubcore_jetty *jetty); +struct ubcore_tjetty *udma_import_jetty(struct ubcore_device *dev, + const struct ubcore_tjetty_cfg *cfg, + struct ubcore_udata *udata); +int udma_unimport_jetty(struct ubcore_tjetty *tjetty); #endif /* _UDMA_JETTY_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c index 86104e17c76f..424ffe425f3d 100644 --- a/drivers/ub/hw/hns3/hns3_udma_main.c +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -342,6 +342,8 @@ static struct ubcore_ops g_udma_dev_ops = { .unimport_jfr = udma_unimport_jfr, .create_jetty = udma_create_jetty, .destroy_jetty = udma_destroy_jetty, + .import_jetty = udma_import_jetty, + .unimport_jetty = udma_unimport_jetty, .create_tp = udma_create_tp, .modify_tp = udma_modify_tp, .destroy_tp = udma_destroy_tp, -- Gitee From a905628c9a6f2eba3c1be0b6818b67c1c014e0bb Mon Sep 17 00:00:00 2001 From: Weibo Zhao Date: Wed, 6 Sep 2023 15:37:29 +0800 Subject: [PATCH 34/42] hns3 udma: support of flush cqe driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA ---------------------------------------------------------- Driver send sq_pi to hardware so hardware can report flush cqes. The user-mode driver invokes user_ctrl to instruct the kernel-mode driver to execute flush_cqe. The kernel-mode driver pass sq_pi to hardware by CMD of modify_qp_to_error. Signed-off-by: Weibo Zhao --- drivers/ub/hw/hns3/hns3_udma_abi.h | 10 +++++ drivers/ub/hw/hns3/hns3_udma_main.c | 67 +++++++++++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_qp.c | 51 ++++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_qp.h | 2 + 4 files changed, 130 insertions(+) diff --git a/drivers/ub/hw/hns3/hns3_udma_abi.h b/drivers/ub/hw/hns3/hns3_udma_abi.h index 4345bf26e4f0..0eabf6d6e6e9 100644 --- a/drivers/ub/hw/hns3/hns3_udma_abi.h +++ b/drivers/ub/hw/hns3/hns3_udma_abi.h @@ -148,4 +148,14 @@ struct udma_create_ctx_resp { uint32_t max_jfs_sge; }; +struct flush_cqe_param { + uint32_t qpn; + uint32_t sq_producer_idx; +}; + +enum udma_user_ctl_handlers { + UDMA_USER_CTL_FLUSH_CQE, + UDMA_OPCODE_NUM, +}; + #endif /* _UDMA_ABI_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c index 424ffe425f3d..b8783b864813 100644 --- a/drivers/ub/hw/hns3/hns3_udma_main.c +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -317,6 +317,72 @@ static int udma_query_device_status(const struct ubcore_device *dev, return 0; } +int udma_user_ctl_flush_cqe(struct ubcore_ucontext *uctx, struct ubcore_user_ctl_in *in, + struct ubcore_user_ctl_out *out, + struct ubcore_udrv_priv *udrv_data) +{ + struct udma_dev *udma_device = to_udma_dev(uctx->ub_dev); + struct flush_cqe_param fcp; + struct udma_qp *udma_qp; + uint32_t sq_pi; + uint32_t qpn; + int ret; + + ret = (int)copy_from_user(&fcp, (void *)in->addr, + sizeof(struct flush_cqe_param)); + if (ret != 0) { + dev_err(udma_device->dev, + "copy_from_user failed in flush_cqe, ret:%d.\n", ret); + return -EFAULT; + } + sq_pi = fcp.sq_producer_idx; + qpn = fcp.qpn; + + xa_lock(&udma_device->qp_table.xa); + udma_qp = (struct udma_qp *)xa_load(&udma_device->qp_table.xa, qpn); + if (!udma_qp) { + dev_err(udma_device->dev, "get qp(0x%x) error.\n", qpn); + xa_unlock(&udma_device->qp_table.xa); + return -EINVAL; + } + refcount_inc(&udma_qp->refcount); + xa_unlock(&udma_device->qp_table.xa); + + ret = udma_flush_cqe(udma_device, udma_qp, sq_pi); + + if (refcount_dec_and_test(&udma_qp->refcount)) + complete(&udma_qp->free); + + return ret; +} + +typedef int (*udma_user_ctl_opcode)(struct ubcore_ucontext *uctx, + struct ubcore_user_ctl_in *in, + struct ubcore_user_ctl_out *out, + struct ubcore_udrv_priv *udrv_data); + +static udma_user_ctl_opcode g_udma_user_ctl_opcodes[] = { + [UDMA_USER_CTL_FLUSH_CQE] = udma_user_ctl_flush_cqe, +}; + +int udma_user_ctl(struct ubcore_user_ctl *k_user_ctl) +{ + struct ubcore_udrv_priv udrv_data = k_user_ctl->udrv_data; + struct ubcore_user_ctl_out out = k_user_ctl->out; + struct ubcore_ucontext *uctx = k_user_ctl->uctx; + struct ubcore_user_ctl_in in = k_user_ctl->in; + struct udma_dev *udma_device; + + udma_device = to_udma_dev(uctx->ub_dev); + if (in.opcode >= UDMA_OPCODE_NUM || + !g_udma_user_ctl_opcodes[in.opcode]) { + dev_err(udma_device->dev, "bad user_ctl opcode: 0x%x.\n", + (int)in.opcode); + return -EINVAL; + } + return g_udma_user_ctl_opcodes[in.opcode](uctx, &in, &out, &udrv_data); +} + static struct ubcore_ops g_udma_dev_ops = { .owner = THIS_MODULE, .abi_version = 1, @@ -347,6 +413,7 @@ static struct ubcore_ops g_udma_dev_ops = { .create_tp = udma_create_tp, .modify_tp = udma_modify_tp, .destroy_tp = udma_destroy_tp, + .user_ctl = udma_user_ctl, }; static void udma_cleanup_uar_table(struct udma_dev *dev) diff --git a/drivers/ub/hw/hns3/hns3_udma_qp.c b/drivers/ub/hw/hns3/hns3_udma_qp.c index 2d27f24ab0ef..7e2bbbf2b094 100644 --- a/drivers/ub/hw/hns3/hns3_udma_qp.c +++ b/drivers/ub/hw/hns3/hns3_udma_qp.c @@ -1628,6 +1628,45 @@ void udma_cleanup_qp_table(struct udma_dev *dev) kfree(dev->qp_table.idx_table.spare_idx); } +int udma_flush_cqe(struct udma_dev *udma_dev, struct udma_qp *udma_qp, + uint32_t sq_pi) +{ + struct udma_qp_context *qp_context; + struct udma_cmd_mailbox *mailbox; + struct udma_qp_context *qpc_mask; + struct udma_cmq_desc desc; + struct udma_mbox *mb; + int ret; + + udma_qp->state = QPS_ERR; + + mailbox = udma_alloc_cmd_mailbox(udma_dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + qp_context = (struct udma_qp_context *)mailbox->buf; + qpc_mask = (struct udma_qp_context *)mailbox->buf + 1; + memset(qpc_mask, 0xff, sizeof(struct udma_qp_context)); + + udma_reg_write(qp_context, QPC_QP_ST, udma_qp->state); + udma_reg_clear(qpc_mask, QPC_QP_ST); + + udma_reg_write(qp_context, QPC_SQ_PRODUCER_IDX, sq_pi); + udma_reg_clear(qpc_mask, QPC_SQ_PRODUCER_IDX); + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mbox_desc_init(mb, mailbox->dma, 0, udma_qp->qpn, UDMA_CMD_MODIFY_QPC); + + ret = udma_cmd_mbox(udma_dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); + if (ret) + dev_err(udma_dev->dev, "flush cqe qp(0x%llx) cmd error(%d).\n", + udma_qp->qpn, ret); + + udma_free_cmd_mailbox(udma_dev, mailbox); + + return ret; +} + void udma_qp_event(struct udma_dev *udma_dev, uint32_t qpn, int event_type) { struct device *dev = udma_dev->dev; @@ -1644,6 +1683,18 @@ void udma_qp_event(struct udma_dev *udma_dev, uint32_t qpn, int event_type) return; } + if (event_type == UDMA_EVENT_TYPE_JFR_LAST_WQE_REACH || + event_type == UDMA_EVENT_TYPE_WQ_CATAS_ERROR || + event_type == UDMA_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR || + event_type == UDMA_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR) { + qp->state = QPS_ERR; + + if (qp->sdb.virt_addr) + qp->sq.head = *(int *)(qp->sdb.virt_addr); + + udma_flush_cqe(udma_dev, qp, qp->sq.head); + } + if (qp->event) qp->event(qp, (enum udma_event)event_type); diff --git a/drivers/ub/hw/hns3/hns3_udma_qp.h b/drivers/ub/hw/hns3/hns3_udma_qp.h index ea5d984ab435..02d61bd37a8c 100644 --- a/drivers/ub/hw/hns3/hns3_udma_qp.h +++ b/drivers/ub/hw/hns3/hns3_udma_qp.h @@ -284,6 +284,8 @@ void init_jetty_x_qpn_bitmap(struct udma_dev *dev, uint32_t jetty_x_shift, uint32_t prefix, uint32_t jid); void clean_jetty_x_qpn_bitmap(struct udma_qpn_bitmap *qpn_map); +int udma_flush_cqe(struct udma_dev *udma_dev, struct udma_qp *udma_qp, + uint32_t sq_pi); void udma_qp_event(struct udma_dev *udma_dev, uint32_t qpn, int event_type); void copy_send_jfc(struct udma_qp *from_qp, struct udma_qp *to_qp); -- Gitee From b3531280f8023528ac34212091e8393f7d24c626 Mon Sep 17 00:00:00 2001 From: Weibo Zhao Date: Thu, 7 Sep 2023 11:48:34 +0800 Subject: [PATCH 35/42] hns3 udma: change the way of modify qp process driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA ------------------------------------------------ This patch change the way of modify qp process. Separate processes like OOR from main process to make it easy to read. Signed-off-by: Weibo Zhao --- drivers/ub/hw/hns3/hns3_udma_jetty.c | 5 +- drivers/ub/hw/hns3/hns3_udma_jfr.c | 6 +- drivers/ub/hw/hns3/hns3_udma_jfs.c | 6 +- drivers/ub/hw/hns3/hns3_udma_qp.c | 391 +++++++++++++++++---------- drivers/ub/hw/hns3/hns3_udma_qp.h | 7 +- drivers/ub/hw/hns3/hns3_udma_tp.c | 26 +- 6 files changed, 280 insertions(+), 161 deletions(-) diff --git a/drivers/ub/hw/hns3/hns3_udma_jetty.c b/drivers/ub/hw/hns3/hns3_udma_jetty.c index 1200c93d65b3..d3c31fbccf20 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jetty.c +++ b/drivers/ub/hw/hns3/hns3_udma_jetty.c @@ -67,6 +67,7 @@ static void udma_fill_jetty_um_qp_attr(struct udma_dev *dev, static int udma_modify_qp_jetty(struct udma_dev *dev, struct udma_jetty *jetty, enum udma_qp_state target_state) { + union ubcore_tp_attr_mask ubcore_attr_mask; struct udma_modify_tp_attr m_attr = {}; struct udma_qp *qp; int ret; @@ -78,8 +79,10 @@ static int udma_modify_qp_jetty(struct udma_dev *dev, struct udma_jetty *jetty, m_attr.path_mtu = UBCORE_MTU_4096; m_attr.hop_limit = MAX_HOP_LIMIT; + ubcore_attr_mask.value = 0; + qp->m_attr = &m_attr; - ret = udma_modify_qp_common(qp, &m_attr, jetty->qp.state, target_state); + ret = udma_modify_qp_common(qp, NULL, ubcore_attr_mask, jetty->qp.state, target_state); if (ret) dev_err(dev->dev, "failed to modify qpc to RTS in Jetty.\n"); diff --git a/drivers/ub/hw/hns3/hns3_udma_jfr.c b/drivers/ub/hw/hns3/hns3_udma_jfr.c index e86c553fb367..59801f84ce7d 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jfr.c +++ b/drivers/ub/hw/hns3/hns3_udma_jfr.c @@ -395,6 +395,7 @@ static void free_jfr_buf(struct udma_dev *dev, struct udma_jfr *jfr) static int udma_modify_jfr_um_qpc(struct udma_dev *dev, struct udma_jfr *jfr, enum udma_qp_state target_state) { + union ubcore_tp_attr_mask ubcore_attr_mask; struct udma_modify_tp_attr attr = {}; struct udma_qp *qp = jfr->um_qp; int ret; @@ -404,7 +405,10 @@ static int udma_modify_jfr_um_qpc(struct udma_dev *dev, struct udma_jfr *jfr, qp->qp_attr.jfr = jfr; qp->recv_jfc = to_udma_jfc(jfr->ubcore_jfr.jfr_cfg.jfc); qp->send_jfc = NULL; - ret = udma_modify_qp_common(qp, &attr, jfr->um_qp->state, target_state); + ubcore_attr_mask.value = 0; + qp->m_attr = &attr; + + ret = udma_modify_qp_common(qp, NULL, ubcore_attr_mask, jfr->um_qp->state, target_state); if (ret) dev_err(dev->dev, "failed to modify qpc to RTR.\n"); diff --git a/drivers/ub/hw/hns3/hns3_udma_jfs.c b/drivers/ub/hw/hns3/hns3_udma_jfs.c index 8ec30057fa7c..7fba63d6f46d 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jfs.c +++ b/drivers/ub/hw/hns3/hns3_udma_jfs.c @@ -39,6 +39,7 @@ static int init_jfs_cfg(struct udma_dev *dev, struct udma_jfs *jfs, static int udma_modify_jfs_um_qp(struct udma_dev *dev, struct udma_jfs *jfs, enum udma_qp_state target_state) { + union ubcore_tp_attr_mask ubcore_attr_mask; struct udma_modify_tp_attr m_attr = {}; struct udma_qp *qp; int ret; @@ -52,7 +53,10 @@ static int udma_modify_jfs_um_qp(struct udma_dev *dev, struct udma_jfs *jfs, m_attr.path_mtu = UBCORE_MTU_4096; m_attr.hop_limit = MAX_HOP_LIMIT; - ret = udma_modify_qp_common(qp, &m_attr, qp->state, target_state); + ubcore_attr_mask.value = 0; + qp->m_attr = &m_attr; + + ret = udma_modify_qp_common(qp, NULL, ubcore_attr_mask, qp->state, target_state); if (ret) dev_err(dev->dev, "failed to modify qpc to RTS.\n"); diff --git a/drivers/ub/hw/hns3/hns3_udma_qp.c b/drivers/ub/hw/hns3/hns3_udma_qp.c index 7e2bbbf2b094..9c70394245cb 100644 --- a/drivers/ub/hw/hns3/hns3_udma_qp.c +++ b/drivers/ub/hw/hns3/hns3_udma_qp.c @@ -205,12 +205,12 @@ static enum udma_mtu to_udma_mtu(enum ubcore_mtu core_mtu) } static inline enum ubcore_mtu get_mtu(struct udma_qp *qp, - const struct udma_modify_tp_attr *attr) + const struct ubcore_tp_attr *attr) { - if (qp->qp_type == QPT_UD) + if (qp->qp_type == QPT_UD || attr == NULL) return UBCORE_MTU_4096; - return attr->path_mtu; + return attr->mtu; } static inline int udma_mtu_enum_to_int(enum ubcore_mtu mtu) @@ -254,6 +254,7 @@ static void udma_free_reorder_cq_buf(struct udma_dev *udma_dev, } static void edit_qpc_for_inline(struct udma_qp_context *context, + struct udma_qp_context *context_mask, struct udma_qp *qp) { struct udma_dev *udma_dev = qp->udma_device; @@ -262,163 +263,123 @@ static void edit_qpc_for_inline(struct udma_qp_context *context, udma_reg_write(context, QPC_CQEIE, !!(udma_dev->caps.flags & UDMA_CAP_FLAG_CQE_INLINE)); + udma_reg_clear(context_mask, QPC_CQEIE); } } -static void edit_qpc_for_ext(struct udma_qp_context *context, - struct udma_qp *qp, - const struct udma_modify_tp_attr *attr) -{ - struct udma_dev *udma_dev = qp->udma_device; - uint16_t dus_regval; - uint16_t aus_regval; - uint16_t real_range; - - if (udma_dev->caps.oor_en) { - udma_reg_enable(&context->ext, QPCEX_OOR_EN); - udma_reg_write(&context->ext, QPCEX_REORDER_CAP, - udma_dev->caps.reorder_cap); - udma_reg_write(&context->ext, QPCEX_ON_FLIGHT_SIZE_L, - udma_dev->caps.onflight_size); - udma_reg_write(&context->ext, QPCEX_ON_FLIGHT_SIZE_H, - udma_dev->caps.onflight_size >> - QPCEX_ON_FLIGHT_SIZE_H_SHIFT); - udma_reg_write(&context->ext, QPCEX_DYN_AT, - udma_dev->caps.dynamic_ack_timeout); - if (udma_dev->caps.flags & UDMA_CAP_FLAG_AR) { - real_range = (attr->udp_range + - UDP_SRCPORT_RANGE_BASE) & - UDP_SRCPORT_RANGE_SIZE_MASK; - dus_regval = attr->data_udp_start & - GENMASK(real_range, 0); - aus_regval = attr->ack_udp_start & - GENMASK(real_range, 0); - - udma_reg_write(&context->ext, QPCEX_AR_EN, attr->ar_en); - udma_reg_write(&context->ext, QPCEX_DATA_UDP_SRCPORT_L, - dus_regval); - udma_reg_write(&context->ext, QPCEX_DATA_UDP_SRCPORT_H, - dus_regval >> - QPCEX_DATA_UDP_SRCPORT_H_SHIFT); - udma_reg_write(&context->ext, QPCEX_ACK_UDP_SRCPORT, - aus_regval); - udma_reg_write(&context->ext, QPCEX_UDP_SRCPORT_RANGE, - attr->udp_range); - } - if (udma_dev->caps.reorder_cq_buffer_en && - qp->qp_attr.reorder_cq_addr) { - udma_reg_enable(&context->ext, QPCEX_REORDER_CQ_EN); - udma_reg_write(&context->ext, QPCEX_REORDER_CQ_ADDR_L, - lower_32_bits(qp->qp_attr.reorder_cq_addr) >> - QPCEX_REORDER_CQ_ADDR_SHIFT); - udma_reg_write(&context->ext, QPCEX_REORDER_CQ_ADDR_H, - upper_32_bits(qp->qp_attr.reorder_cq_addr)); - udma_reg_write(&context->ext, QPCEX_REORDER_CQ_SHIFT, - udma_dev->caps.reorder_cq_shift); - } - } - udma_reg_write(&context->ext, QPCEX_RTT, QPCEX_RTT_INIT); - udma_reg_write(&context->ext, QPCEX_P_TYPE, QPCEX_P_TYPE_UDMA); -} - -static void edit_qpc_for_db(struct udma_qp_context *context, +static void edit_qpc_for_db(struct udma_qp_context *context, struct udma_qp_context *context_mask, struct udma_qp *qp) { - if (qp->en_flags & UDMA_QP_CAP_RQ_RECORD_DB) + if (qp->en_flags & UDMA_QP_CAP_RQ_RECORD_DB) { udma_reg_enable(context, QPC_RQ_RECORD_EN); + udma_reg_clear(context_mask, QPC_RQ_RECORD_EN); + } - if (qp->en_flags & UDMA_QP_CAP_OWNER_DB) + if (qp->en_flags & UDMA_QP_CAP_OWNER_DB) { udma_reg_enable(context, QPC_OWNER_MODE); + udma_reg_clear(context_mask, QPC_OWNER_MODE); + } } static void edit_qpc_for_srqn(struct udma_qp *qp, - struct udma_qp_context *context) + struct udma_qp_context *context, + struct udma_qp_context *context_mask) { if (qp->qp_attr.jfr) { udma_reg_enable(context, QPC_SRQ_EN); + udma_reg_clear(context_mask, QPC_SRQ_EN); + udma_reg_write(context, QPC_SRQN, qp->qp_attr.jfr->jfrn); + udma_reg_clear(context_mask, QPC_SRQN); } } static void edit_qpc_for_rxcqn(struct udma_qp *qp, - struct udma_qp_context *context) + struct udma_qp_context *context, + struct udma_qp_context *context_mask) { - if (qp->recv_jfc) + if (qp->recv_jfc) { udma_reg_write(context, QPC_RX_CQN, qp->recv_jfc->cqn); -} - -static void edit_qpc_for_psn(const struct udma_modify_tp_attr *attr, - struct udma_qp_context *context) -{ - udma_reg_write(context, QPC_RX_REQ_EPSN, attr->rq_psn); - udma_reg_write(context, QPC_RAQ_PSN, attr->rq_psn - 1); - udma_reg_write(context, QPC_SQ_CUR_PSN, attr->sq_psn); - udma_reg_write(context, QPC_SQ_MAX_PSN, attr->sq_psn); + udma_reg_clear(context_mask, QPC_RX_CQN); + } } static void edit_qpc_for_retransmission_parm(struct udma_dev *udma_device, struct udma_qp *qp, const struct udma_modify_tp_attr *attr, - struct udma_qp_context *context) + struct udma_qp_context *context, + struct udma_qp_context *context_mask) { if (qp->qp_type != QPT_UD) { udma_reg_write(context, QPC_MIN_RNR_TIME, attr->min_rnr_timer); + udma_reg_clear(context_mask, QPC_MIN_RNR_TIME); + udma_reg_write(context, QPC_RETRY_CNT, attr->retry_cnt); + udma_reg_clear(context_mask, QPC_RETRY_CNT); + udma_reg_write(context, QPC_RETRY_NUM_INIT, attr->retry_cnt); + udma_reg_clear(context_mask, QPC_RETRY_NUM_INIT); + udma_reg_write(context, QPC_RNR_CNT, attr->rnr_retry); + udma_reg_clear(context_mask, QPC_RNR_CNT); + udma_reg_write(context, QPC_RNR_NUM_INIT, attr->rnr_retry); + udma_reg_clear(context_mask, QPC_RNR_NUM_INIT); - if (check_qp_timeout_cfg_range(udma_device, &attr->ack_timeout)) + if (check_qp_timeout_cfg_range(udma_device, &attr->ack_timeout)) { udma_reg_write(context, QPC_AT, attr->ack_timeout); + udma_reg_clear(context_mask, QPC_AT); + } } } static void edit_qpc_for_write(struct udma_qp *qp, - struct udma_qp_context *context) + struct udma_qp_context *context, + struct udma_qp_context *context_mask) { udma_reg_enable(context, QPC_FLUSH_EN); + udma_reg_clear(context_mask, QPC_FLUSH_EN); + udma_reg_enable(context, QPC_AW_EN); + udma_reg_clear(context_mask, QPC_AW_EN); + udma_reg_enable(context, QPC_WN_EN); + udma_reg_clear(context_mask, QPC_WN_EN); + udma_reg_enable(context, QPC_RMT_E2E); + udma_reg_clear(context_mask, QPC_RMT_E2E); + udma_reg_write(context, QPC_SIG_TYPE, SIGNAL_REQ_WR); + udma_reg_clear(context_mask, QPC_SIG_TYPE); } static void edit_qpc_for_receive(struct udma_qp *qp, const struct udma_modify_tp_attr *attr, - struct udma_qp_context *context) + struct udma_qp_context *context, + struct udma_qp_context *context_mask) { - uint8_t lp_pktn_ini; uint8_t *dmac; - udma_reg_write(context, QPC_DQPN, attr->dest_qp_num); - dmac = (uint8_t *)attr->dmac; memcpy(&context->dmac, dmac, sizeof(uint32_t)); udma_reg_write(context, QPC_DMAC_L, *((uint32_t *)(&dmac[0]))); + udma_reg_clear(context_mask, QPC_DMAC_L); + udma_reg_write(context, QPC_DMAC_H, *((uint16_t *)(&dmac[QPC_DMAC_H_IDX]))); - - qp->ubcore_path_mtu = get_mtu(qp, attr); - qp->path_mtu = to_udma_mtu(qp->ubcore_path_mtu); - udma_reg_write(context, QPC_MTU, qp->path_mtu); - - /* MTU * (2 ^ LP_PKTN_INI) shouldn't be bigger than 16KB */ - lp_pktn_ini = ilog2(MAX_LP_MSG_LEN / udma_mtu_enum_to_int(qp->path_mtu)); - - udma_reg_write(context, QPC_LP_PKTN_INI, lp_pktn_ini); - - /* ACK_REQ_FREQ should be larger than or equal to LP_PKTN_INI */ - udma_reg_write(context, QPC_ACK_REQ_FREQ, lp_pktn_ini); + udma_reg_clear(context_mask, QPC_DMAC_H); context->rq_rnr_timer = 0; + context_mask->rq_rnr_timer = 0; /* rocee send 2^lp_sgen_ini segs every time */ udma_reg_write(context, QPC_LP_SGEN_INI, SGEN_INI_VALUE); + udma_reg_clear(context_mask, QPC_LP_SGEN_INI); } static int modify_qp_reset_to_rtr(struct udma_qp *qp, @@ -429,37 +390,41 @@ static int modify_qp_reset_to_rtr(struct udma_qp *qp, struct udma_dev *udma_device = qp->udma_device; udma_reg_write(context, QPC_RRE, 1); + udma_reg_clear(context_mask, QPC_RRE); + udma_reg_write(context, QPC_RWE, 1); + udma_reg_clear(context_mask, QPC_RWE); udma_reg_write(context, QPC_TST, qp->qp_type); + udma_reg_clear(context_mask, QPC_TST); udma_reg_write(context, QPC_RQWS, ilog2(qp->rq.max_gs)); + udma_reg_clear(context_mask, QPC_RQWS); /* No VLAN need to set 0xFFF */ udma_reg_write(context, QPC_VLAN_ID, 0xfff); + udma_reg_clear(context_mask, QPC_VLAN_ID); - edit_qpc_for_db(context, qp); + edit_qpc_for_db(context, context_mask, qp); - edit_qpc_for_inline(context, qp); + edit_qpc_for_inline(context, context_mask, qp); - edit_qpc_for_ext(context, qp, attr); - - edit_qpc_for_srqn(qp, context); + udma_reg_write(&context->ext, QPCEX_P_TYPE, QPCEX_P_TYPE_UDMA); + udma_reg_clear(&context_mask->ext, QPCEX_P_TYPE); - edit_qpc_for_psn(attr, context); + edit_qpc_for_srqn(qp, context, context_mask); - edit_qpc_for_retransmission_parm(udma_device, qp, attr, context); + edit_qpc_for_retransmission_parm(udma_device, qp, attr, context, context_mask); - edit_qpc_for_rxcqn(qp, context); + edit_qpc_for_rxcqn(qp, context, context_mask); /* * Enable atomic WRITE and persistence WRITE and Write With Notify * operations in QPC when modify_qp_init_to_rtr. */ - edit_qpc_for_write(qp, context); - - edit_qpc_for_receive(qp, attr, context); + edit_qpc_for_write(qp, context, context_mask); + edit_qpc_for_receive(qp, attr, context, context_mask); return 0; } @@ -489,40 +454,161 @@ static int modify_qp_rtr_to_rts(struct udma_qp *qp, return 0; } -static int udma_set_opt_fields(struct udma_qp *qp, - const struct udma_modify_tp_attr *attr, - struct udma_qp_context *context, - struct udma_qp_context *context_mask) +static void udma_set_spray_field(struct udma_qp *qp, + const struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask ubcore_mask, + struct udma_qp_context *context, + struct udma_qp_context *context_mask) { - int ret = 0; + struct udma_dev *udma_dev = qp->udma_device; + struct udma_modify_tp_attr *m_attr; + uint16_t dus_regval; + uint16_t aus_regval; + uint16_t real_range; + + m_attr = qp->m_attr; + real_range = (m_attr->udp_range + + UDP_SRCPORT_RANGE_BASE) & + UDP_SRCPORT_RANGE_SIZE_MASK; + dus_regval = m_attr->data_udp_start & + GENMASK(real_range, 0); + aus_regval = m_attr->ack_udp_start & + GENMASK(real_range, 0); + + udma_reg_enable(&context->ext, QPCEX_AR_EN); + udma_reg_clear(&context_mask->ext, QPCEX_AR_EN); + + udma_reg_write(&context->ext, QPCEX_ACK_UDP_SRCPORT, + aus_regval); + udma_reg_clear(&context_mask->ext, QPCEX_ACK_UDP_SRCPORT); + + udma_reg_write(&context->ext, QPCEX_DATA_UDP_SRCPORT_L, + dus_regval); + udma_reg_clear(&context_mask->ext, QPCEX_DATA_UDP_SRCPORT_L); + + udma_reg_write(&context->ext, QPCEX_DATA_UDP_SRCPORT_H, + dus_regval >> + QPCEX_DATA_UDP_SRCPORT_H_SHIFT); + udma_reg_clear(&context_mask->ext, QPCEX_DATA_UDP_SRCPORT_H); + + udma_reg_write(&context->ext, QPCEX_UDP_SRCPORT_RANGE, + m_attr->udp_range); + udma_reg_clear(&context_mask->ext, QPCEX_UDP_SRCPORT_RANGE); + + if (udma_dev->caps.reorder_cq_buffer_en && + qp->qp_attr.reorder_cq_addr) { + udma_reg_enable(&context->ext, QPCEX_REORDER_CQ_EN); + udma_reg_clear(&context_mask->ext, QPCEX_REORDER_CQ_EN); + + udma_reg_write(&context->ext, QPCEX_REORDER_CQ_ADDR_L, + lower_32_bits(qp->qp_attr.reorder_cq_addr) >> + QPCEX_REORDER_CQ_ADDR_SHIFT); + udma_reg_clear(&context_mask->ext, QPCEX_REORDER_CQ_ADDR_L); + + udma_reg_write(&context->ext, QPCEX_REORDER_CQ_ADDR_H, + upper_32_bits(qp->qp_attr.reorder_cq_addr)); + udma_reg_clear(&context_mask->ext, QPCEX_REORDER_CQ_ADDR_H); + + udma_reg_write(&context->ext, QPCEX_REORDER_CQ_SHIFT, + udma_dev->caps.reorder_cq_shift); + udma_reg_clear(&context_mask->ext, QPCEX_REORDER_CQ_SHIFT); + } +} + +static void udma_set_oor_field(struct udma_qp *qp, + const struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask ubcore_mask, + struct udma_qp_context *context, + struct udma_qp_context *context_mask) +{ + struct udma_dev *udma_dev = qp->udma_device; + + udma_reg_enable(&context->ext, QPCEX_OOR_EN); + udma_reg_clear(&context_mask->ext, QPCEX_OOR_EN); + + udma_reg_write(&context->ext, QPCEX_REORDER_CAP, + udma_dev->caps.reorder_cap); + udma_reg_clear(&context_mask->ext, QPCEX_REORDER_CAP); + + udma_reg_write(&context->ext, QPCEX_ON_FLIGHT_SIZE_L, + udma_dev->caps.onflight_size); + udma_reg_clear(&context_mask->ext, QPCEX_ON_FLIGHT_SIZE_L); + + udma_reg_write(&context->ext, QPCEX_ON_FLIGHT_SIZE_H, + udma_dev->caps.onflight_size >> + QPCEX_ON_FLIGHT_SIZE_H_SHIFT); + udma_reg_clear(&context_mask->ext, QPCEX_ON_FLIGHT_SIZE_H); + + udma_reg_write(&context->ext, QPCEX_DYN_AT, + udma_dev->caps.dynamic_ack_timeout); + udma_reg_clear(&context_mask->ext, QPCEX_DYN_AT); +} + +static void udma_set_opt_fields(struct udma_qp *qp, + const struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask ubcore_mask, + struct udma_qp_context *context, + struct udma_qp_context *context_mask) +{ + struct udma_dev *udma_dev = qp->udma_device; + uint8_t lp_pktn_ini; if (attr == NULL) - goto out; + return; + + if (ubcore_mask.bs.flag && attr->flag.bs.oor_en && udma_dev->caps.oor_en) + udma_set_oor_field(qp, attr, ubcore_mask, context, context_mask); - if (attr->max_dest_rd_atomic) { - udma_reg_write(context, QPC_RR_MAX, - fls(attr->max_dest_rd_atomic - 1)); - udma_reg_clear(context_mask, QPC_RR_MAX); + if (ubcore_mask.bs.flag && attr->flag.bs.spray_en && + (udma_dev->caps.flags & UDMA_CAP_FLAG_AR)) + udma_set_spray_field(qp, attr, ubcore_mask, context, context_mask); + + if (ubcore_mask.bs.peer_tpn) { + udma_reg_write(context, QPC_DQPN, attr->peer_tpn); + udma_reg_clear(context_mask, QPC_DQPN); } - if (attr->max_rd_atomic) { - udma_reg_write(context, QPC_SR_MAX, - fls(attr->max_rd_atomic - 1)); - udma_reg_clear(context_mask, QPC_SR_MAX); + if (ubcore_mask.bs.tx_psn) { + udma_reg_write(context, QPC_RX_ACK_EPSN, attr->tx_psn); + udma_reg_write(context, QPC_RETRY_MSG_PSN_L, attr->tx_psn); + udma_reg_write(context, QPC_RETRY_MSG_PSN_H, + attr->tx_psn >> RETRY_MSG_PSN_H_OFFSET); + udma_reg_write(context, QPC_RETRY_MSG_FPKT_PSN, attr->tx_psn); + udma_reg_write(context, QPC_SQ_CUR_PSN, attr->tx_psn); + udma_reg_write(context, QPC_SQ_MAX_PSN, attr->tx_psn); + + udma_reg_clear(context_mask, QPC_RX_ACK_EPSN); + udma_reg_clear(context_mask, QPC_RETRY_MSG_PSN_L); + udma_reg_clear(context_mask, QPC_RETRY_MSG_PSN_H); + udma_reg_clear(context_mask, QPC_RETRY_MSG_FPKT_PSN); + udma_reg_clear(context_mask, QPC_SQ_CUR_PSN); + udma_reg_clear(context_mask, QPC_SQ_MAX_PSN); } - udma_reg_write(context, QPC_RX_ACK_EPSN, attr->sq_psn); - udma_reg_write(context, QPC_RETRY_MSG_PSN_H, attr->sq_psn); - udma_reg_write(context, QPC_RETRY_NUM_INIT, attr->retry_cnt); - udma_reg_write(context, QPC_RETRY_MSG_FPKT_PSN, attr->sq_psn); + if (ubcore_mask.bs.rx_psn) { + udma_reg_write(context, QPC_RX_REQ_EPSN, attr->rx_psn); + udma_reg_write(context, QPC_RAQ_PSN, attr->rx_psn - 1); - udma_reg_clear(context_mask, QPC_RX_ACK_EPSN); - udma_reg_clear(context_mask, QPC_RETRY_MSG_PSN_H); - udma_reg_clear(context_mask, QPC_RETRY_NUM_INIT); - udma_reg_clear(context_mask, QPC_RETRY_MSG_FPKT_PSN); + udma_reg_clear(context_mask, QPC_RX_REQ_EPSN); + udma_reg_clear(context_mask, QPC_RAQ_PSN); + } -out: - return ret; + if (ubcore_mask.bs.mtu) { + qp->ubcore_path_mtu = get_mtu(qp, attr); + qp->path_mtu = to_udma_mtu(qp->ubcore_path_mtu); + udma_reg_write(context, QPC_MTU, qp->path_mtu); + udma_reg_clear(context_mask, QPC_MTU); + + /* MTU * (2 ^ LP_PKTN_INI) shouldn't be bigger than 16KB */ + lp_pktn_ini = ilog2(MAX_LP_MSG_LEN / udma_mtu_enum_to_int(qp->path_mtu)); + + udma_reg_write(context, QPC_LP_PKTN_INI, lp_pktn_ini); + udma_reg_clear(context_mask, QPC_LP_PKTN_INI); + + /* ACK_REQ_FREQ should be larger than or equal to LP_PKTN_INI */ + udma_reg_write(context, QPC_ACK_REQ_FREQ, lp_pktn_ini); + udma_reg_clear(context_mask, QPC_ACK_REQ_FREQ); + } } static int udma_set_abs_fields(struct udma_qp *qp, @@ -536,7 +622,6 @@ static int udma_set_abs_fields(struct udma_qp *qp, int ret = 0; if (curr_state == QPS_RESET && new_state == QPS_RTR) { - memset(context_mask, 0, udma_device->caps.qpc_sz); ret = modify_qp_reset_to_rtr(qp, attr, context, context_mask); if (ret) { dev_err(udma_device->dev, @@ -544,7 +629,6 @@ static int udma_set_abs_fields(struct udma_qp *qp, goto out; } } else if (curr_state == QPS_RESET && new_state == QPS_RTS) { - memset(context_mask, 0, udma_device->caps.qpc_sz); ret = modify_qp_reset_to_rtr(qp, attr, context, context_mask); if (ret) { dev_err(udma_device->dev, @@ -570,8 +654,31 @@ static int udma_set_abs_fields(struct udma_qp *qp, return ret; } +static void udma_set_um_attr(struct udma_qp *qp, + struct udma_qp_context *context, + struct udma_qp_context *context_mask) +{ + uint8_t lp_pktn_ini; + + qp->ubcore_path_mtu = get_mtu(qp, NULL); + qp->path_mtu = to_udma_mtu(qp->ubcore_path_mtu); + udma_reg_write(context, QPC_MTU, qp->path_mtu); + udma_reg_clear(context_mask, QPC_MTU); + + /* MTU * (2 ^ LP_PKTN_INI) shouldn't be bigger than 16KB */ + lp_pktn_ini = ilog2(MAX_LP_MSG_LEN / udma_mtu_enum_to_int(qp->path_mtu)); + + udma_reg_write(context, QPC_LP_PKTN_INI, lp_pktn_ini); + udma_reg_clear(context_mask, QPC_LP_PKTN_INI); + + /* ACK_REQ_FREQ should be larger than or equal to LP_PKTN_INI */ + udma_reg_write(context, QPC_ACK_REQ_FREQ, lp_pktn_ini); + udma_reg_clear(context_mask, QPC_ACK_REQ_FREQ); +} + int udma_modify_qp_common(struct udma_qp *qp, - const struct udma_modify_tp_attr *attr, + const struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask ubcore_mask, enum udma_qp_state curr_state, enum udma_qp_state new_state) { @@ -581,20 +688,19 @@ int udma_modify_qp_common(struct udma_qp *qp, struct udma_qp_context *context_mask = ctx + 1; int ret = 0; + memset(context, 0, udma_device->caps.qpc_sz); memset(context_mask, 0xff, udma_device->caps.qpc_sz); if (new_state != QPS_RESET) { - ret = udma_set_abs_fields(qp, attr, curr_state, new_state, + ret = udma_set_abs_fields(qp, qp->m_attr, curr_state, new_state, context, context_mask); if (ret) goto out; } - ret = udma_set_opt_fields(qp, attr, context, context_mask); - if (ret) { - dev_err(udma_device->dev, "failed to set option fields, ret = %d.\n", - ret); - goto out; - } + if (qp->qp_type == QPT_UD) + udma_set_um_attr(qp, context, context_mask); + + udma_set_opt_fields(qp, attr, ubcore_mask, context, context_mask); udma_reg_write(context, QPC_INV_CREDIT, (!!qp->qp_attr.jfr) ? 1 : 0); udma_reg_clear(context_mask, QPC_INV_CREDIT); @@ -602,7 +708,7 @@ int udma_modify_qp_common(struct udma_qp *qp, udma_reg_write(context, QPC_QP_ST, new_state); udma_reg_clear(context_mask, QPC_QP_ST); - udma_set_path(attr, context, context_mask); + udma_set_path(qp->m_attr, context, context_mask); udma_reg_write(&context->ext, QPCEX_P_TYPE, QPCEX_P_TYPE_UDMA); udma_reg_clear(&context_mask->ext, QPCEX_P_TYPE); @@ -1473,6 +1579,7 @@ int udma_create_qp_common(struct udma_dev *udma_dev, struct udma_qp *qp, struct ubcore_udata *udata) { struct udma_qp_attr *qp_attr = &qp->qp_attr; + struct udma_qp_context ctx[2] = {0}; struct device *dev = udma_dev->dev; struct udma_create_tp_ucmd ucmd; struct udma_create_tp_resp resp; @@ -1554,6 +1661,12 @@ int udma_create_qp_common(struct udma_dev *udma_dev, struct udma_qp *qp, refcount_set(&qp->refcount, 1); init_completion(&qp->free); + ret = udma_pass_qpc_to_hw(udma_dev, ctx, ctx+1, qp); + if (ret) { + dev_err(dev, "failed to pass QPC to HW, ret = %d.\n", ret); + goto err_copy; + } + return 0; err_copy: diff --git a/drivers/ub/hw/hns3/hns3_udma_qp.h b/drivers/ub/hw/hns3/hns3_udma_qp.h index 02d61bd37a8c..86a74e106617 100644 --- a/drivers/ub/hw/hns3/hns3_udma_qp.h +++ b/drivers/ub/hw/hns3/hns3_udma_qp.h @@ -27,6 +27,7 @@ #define QPC_DMAC_H_IDX 4 #define UDP_RANGE_BASE 8 #define UDMA_SQ_WQE_SHIFT 6 +#define RETRY_MSG_PSN_H_OFFSET 16 struct udma_qp_context_ex { uint32_t data[64]; @@ -101,10 +102,10 @@ struct udma_qp_context { #define QPC_OWNER_MODE QPC_FIELD_LOC(1536, 1536) #define QPC_SQ_MAX_PSN QPC_FIELD_LOC(1567, 1544) #define QPC_RMT_E2E QPC_FIELD_LOC(1660, 1660) -#define QPC_SR_MAX QPC_FIELD_LOC(1663, 1661) #define QPC_RETRY_NUM_INIT QPC_FIELD_LOC(1690, 1688) #define QPC_RETRY_CNT QPC_FIELD_LOC(1695, 1693) #define QPC_RETRY_MSG_MSN QPC_FIELD_LOC(1743, 1728) +#define QPC_RETRY_MSG_PSN_L QPC_FIELD_LOC(1759, 1744) #define QPC_RETRY_MSG_PSN_H QPC_FIELD_LOC(1767, 1760) #define QPC_RETRY_MSG_FPKT_PSN QPC_FIELD_LOC(1791, 1768) #define QPC_RX_SQ_CUR_BLK_ADDR_L QPC_FIELD_LOC(1823, 1792) @@ -266,12 +267,14 @@ struct udma_qp { uint8_t priority; bool no_free_wqe_buf; bool force_free_wqe_buf; + struct udma_modify_tp_attr *m_attr; }; #define gen_qpn(high, mid, low) ((high) | (mid) | (low)) int udma_modify_qp_common(struct udma_qp *qp, - const struct udma_modify_tp_attr *attr, + const struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask ubcore_mask, enum udma_qp_state curr_state, enum udma_qp_state new_state); int udma_fill_qp_attr(struct udma_dev *udma_dev, struct udma_qp_attr *qp_attr, diff --git a/drivers/ub/hw/hns3/hns3_udma_tp.c b/drivers/ub/hw/hns3/hns3_udma_tp.c index 2372db943dc2..8178eb1ea44c 100644 --- a/drivers/ub/hw/hns3/hns3_udma_tp.c +++ b/drivers/ub/hw/hns3/hns3_udma_tp.c @@ -48,10 +48,9 @@ struct udma_modify_tp_attr *udma_get_m_attr(struct ubcore_tp *tp, struct udma_qp return NULL; memcpy(m_attr->dmac, tp->peer_net_addr.mac, sizeof(m_attr->dmac)); - - m_attr->max_dest_rd_atomic = 0; - m_attr->max_rd_atomic = 0; - m_attr->rq_psn = tp->rx_psn; + m_attr->data_udp_start = tp->data_udp_start; + m_attr->ack_udp_start = tp->ack_udp_start; + m_attr->udp_range = tp->udp_range; m_attr->hop_limit = MAX_HOP_LIMIT; m_attr->sgid_index = 0; *(uint32_t *)(&m_attr->dipv4) = *(uint32_t *)(tp->peer_eid.raw + @@ -75,17 +74,6 @@ struct udma_modify_tp_attr *udma_get_m_attr(struct ubcore_tp *tp, struct udma_qp } } - if (mask.bs.peer_tpn) - m_attr->dest_qp_num = attr->peer_tpn; - - if (mask.bs.tx_psn) - m_attr->sq_psn = attr->tx_psn; - else - m_attr->sq_psn = tp->tx_psn; - - if (mask.bs.mtu) - m_attr->path_mtu = attr->mtu; - return m_attr; } @@ -121,9 +109,10 @@ int udma_modify_tp(struct ubcore_tp *tp, const struct ubcore_tp_attr *attr, qp->udma_device = udma_device; qp->send_jfc = qp->qp_attr.send_jfc; qp->recv_jfc = qp->qp_attr.recv_jfc; + qp->m_attr = m_attr; if (attr) qp->ubcore_path_mtu = attr->mtu; - ret = udma_modify_qp_common(qp, m_attr, curr_state, target_state); + ret = udma_modify_qp_common(qp, attr, mask, curr_state, target_state); kfree(m_attr); error: return ret; @@ -177,6 +166,7 @@ void *udma_erase_tp(struct udma_tp *udma_tp) int udma_destroy_tp(struct ubcore_tp *tp) { struct udma_dev *udma_device = to_udma_dev(tp->ub_dev); + union ubcore_tp_attr_mask ubcore_attr_mask; enum udma_qp_state curr_state; struct udma_tp *udma_tp; struct udma_qp *qp; @@ -191,9 +181,11 @@ int udma_destroy_tp(struct ubcore_tp *tp) qp = &udma_tp->qp; curr_state = to_udma_qp_state(tp->state); + ubcore_attr_mask.value = 0; + qp->m_attr = NULL; if (qp->state != QPS_RESET) { - ret = udma_modify_qp_common(qp, NULL, curr_state, QPS_RESET); + ret = udma_modify_qp_common(qp, NULL, ubcore_attr_mask, curr_state, QPS_RESET); if (ret) { dev_err(udma_device->dev, "Modify QP 0x%06llx to Reset failed(%d).\n", -- Gitee From 178ed4fc47928eb1d29986b848031c6ae2c281f7 Mon Sep 17 00:00:00 2001 From: Weibo Zhao Date: Tue, 12 Sep 2023 15:05:46 +0800 Subject: [PATCH 36/42] hns3 udma: support POE mode. driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA ----------------------------------------------------- This patch support POE mode of hns3-udma. POE mode is a way that jfce was report to STARS directly. In POE mode, doorbell address was passed to STARS so that STARS can knock doorbell whenever it wants. Signed-off-by: Weibo Zhao --- drivers/ub/hw/hns3/hns3_udma_abi.h | 14 +- drivers/ub/hw/hns3/hns3_udma_hw.h | 15 ++ drivers/ub/hw/hns3/hns3_udma_jfc.c | 25 +++- drivers/ub/hw/hns3/hns3_udma_main.c | 208 ++++++++++++++++++++++++++++ 4 files changed, 259 insertions(+), 3 deletions(-) diff --git a/drivers/ub/hw/hns3/hns3_udma_abi.h b/drivers/ub/hw/hns3/hns3_udma_abi.h index 0eabf6d6e6e9..819aef32323c 100644 --- a/drivers/ub/hw/hns3/hns3_udma_abi.h +++ b/drivers/ub/hw/hns3/hns3_udma_abi.h @@ -36,10 +36,11 @@ enum { }; enum udma_jfc_init_attr_mask { - UDMA_JFC_NOTIFY_CREATE_FLAGS = 1 << 0, + UDMA_JFC_NOTIFY_OR_POE_CREATE_FLAGS = 1 << 0, }; enum udma_jfc_create_flags { + UDMA_JFC_CREATE_ENABLE_POE_MODE = 1 << 0, UDMA_JFC_CREATE_ENABLE_NOTIFY = 1 << 1, }; @@ -67,6 +68,7 @@ struct udma_create_jfr_resp { struct udma_jfc_attr_ex { uint64_t jfc_ex_mask; /* Use enum udma_jfc_init_attr_mask */ uint64_t create_flags; /* Use enum udma_jfc_create_flags */ + uint8_t poe_channel; /* poe channel to use */ uint64_t notify_addr; uint8_t notify_mode; /* Use enum udma_jfc_notify_mode */ }; @@ -146,6 +148,8 @@ struct udma_create_ctx_resp { uint32_t max_jfr_sge; uint32_t max_jfs_wr; uint32_t max_jfs_sge; + uint32_t poe_ch_num; + uint64_t db_addr; }; struct flush_cqe_param { @@ -153,8 +157,16 @@ struct flush_cqe_param { uint32_t sq_producer_idx; }; +struct udma_poe_info { + uint8_t en; + uint8_t poe_channel; + uint64_t poe_addr; +}; + enum udma_user_ctl_handlers { UDMA_USER_CTL_FLUSH_CQE, + UDMA_CONFIG_POE_CHANNEL, + UDMA_QUERY_POE_CHANNEL, UDMA_OPCODE_NUM, }; diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.h b/drivers/ub/hw/hns3/hns3_udma_hw.h index 5bc0e119bd64..e458f6778cce 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hw.h +++ b/drivers/ub/hw/hns3/hns3_udma_hw.h @@ -229,6 +229,8 @@ enum udma_opcode_type { UDMA_QUERY_OOR_CAPS = 0xA002, UDMA_OPC_DEID_TBL_ADD = 0xA110, UDMA_OPC_CFG_GMV_TBL = 0xA140, + UDMA_OPC_CFG_POE_ADDR = 0x801B, + UDMA_OPC_CFG_POE_ATTR = 0x801C, UDMA_OPC_QUERY_PORT_INFO = 0x7104, }; @@ -482,6 +484,19 @@ union udma_eid { } bit32_data; }; +struct udma_poe_cfg_addr_cmq { + uint32_t channel_id; + uint32_t poe_addr_l; + uint32_t poe_addr_h; + uint32_t rsv[3]; +}; + +struct udma_poe_cfg_attr_cmq { + uint32_t channel_id; + uint32_t rsv_en_outstd; + uint32_t rsv[4]; +}; + struct udma_port_info_cmq { uint32_t speed; uint8_t query_type; diff --git a/drivers/ub/hw/hns3/hns3_udma_jfc.c b/drivers/ub/hw/hns3/hns3_udma_jfc.c index 5be01495c036..f4263e82e46b 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jfc.c +++ b/drivers/ub/hw/hns3/hns3_udma_jfc.c @@ -56,6 +56,17 @@ static int check_jfc_cfg(struct udma_dev *udma_dev, const struct ubcore_jfc_cfg return 0; } +static int check_poe_attr(struct udma_dev *udma_dev, + struct udma_jfc_attr_ex *jfc_attr_ex) +{ + if (!(udma_dev->caps.flags & UDMA_CAP_FLAG_POE)) { + dev_err(udma_dev->dev, "Unsupport POE JFC.\n"); + return -EINVAL; + } + + return 0; +} + static int check_notify_attr(struct udma_dev *udma_dev, struct udma_jfc_attr_ex *jfc_attr_ex) { @@ -94,6 +105,9 @@ static int check_jfc_attr_ex(struct udma_dev *udma_dev, int ret; switch (jfc_attr_ex->create_flags) { + case UDMA_JFC_CREATE_ENABLE_POE_MODE: + ret = check_poe_attr(udma_dev, jfc_attr_ex); + break; case UDMA_JFC_CREATE_ENABLE_NOTIFY: ret = check_notify_attr(udma_dev, jfc_attr_ex); break; @@ -132,7 +146,7 @@ static int check_create_jfc(struct udma_dev *udma_dev, } if (ucmd->jfc_attr_ex.jfc_ex_mask & - UDMA_JFC_NOTIFY_CREATE_FLAGS) { + UDMA_JFC_NOTIFY_OR_POE_CREATE_FLAGS) { ret = check_jfc_attr_ex(udma_dev, &ucmd->jfc_attr_ex); if (ret) { dev_err(udma_dev->dev, @@ -155,7 +169,7 @@ static void init_jfc(struct udma_jfc *udma_jfc, struct udma_create_jfc_ucmd *ucm spin_lock_init(&udma_jfc->lock); INIT_LIST_HEAD(&udma_jfc->sq_list); INIT_LIST_HEAD(&udma_jfc->rq_list); - if (ucmd->jfc_attr_ex.jfc_ex_mask & UDMA_JFC_NOTIFY_CREATE_FLAGS) + if (ucmd->jfc_attr_ex.jfc_ex_mask & UDMA_JFC_NOTIFY_OR_POE_CREATE_FLAGS) udma_jfc->jfc_attr_ex = ucmd->jfc_attr_ex; } @@ -315,6 +329,13 @@ static void udma_write_jfc_cqc(struct udma_dev *udma_dev, struct udma_jfc *udma_ upper_32_bits(udma_jfc->db.dma)); } + if (udma_jfc->jfc_attr_ex.create_flags == + UDMA_JFC_CREATE_ENABLE_POE_MODE) { + udma_reg_enable(jfc_context, CQC_POE_EN); + udma_reg_write(jfc_context, CQC_POE_NUM, + udma_jfc->jfc_attr_ex.poe_channel); + } + if (udma_jfc->jfc_attr_ex.create_flags == UDMA_JFC_CREATE_ENABLE_NOTIFY) set_write_notify_param(udma_dev, udma_jfc, jfc_context); } diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c index b8783b864813..a29680132fa6 100644 --- a/drivers/ub/hw/hns3/hns3_udma_main.c +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -85,6 +85,9 @@ static int udma_init_ctx_resp(struct udma_dev *dev, struct ubcore_udrv_priv *udr resp.max_jfr_sge = dev->caps.max_srq_sges; resp.max_jfs_wr = dev->caps.max_wqes; resp.max_jfs_sge = dev->caps.max_sq_sg; + resp.poe_ch_num = dev->caps.poe_ch_num; + resp.db_addr = pci_resource_start(dev->pci_dev, UDMA_DEV_START_OFFSET) + + UDMA_DB_ADDR_OFFSET; ret = copy_to_user((void *)udrv_data->out_addr, &resp, min(udrv_data->out_len, (uint32_t)sizeof(resp))); @@ -356,6 +359,202 @@ int udma_user_ctl_flush_cqe(struct ubcore_ucontext *uctx, struct ubcore_user_ctl return ret; } +static int config_poe_addr(struct udma_dev *udma_device, uint8_t id, + uint64_t addr) +{ + struct udma_poe_cfg_addr_cmq *cmd; + struct udma_cmq_desc desc; + int ret; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_CFG_POE_ADDR, false); + cmd = (struct udma_poe_cfg_addr_cmq *)desc.data; + cmd->channel_id = cpu_to_le32(id); + cmd->poe_addr_l = cpu_to_le32(lower_32_bits(addr)); + cmd->poe_addr_h = cpu_to_le32(upper_32_bits(addr)); + + ret = udma_cmq_send(udma_device, &desc, 1); + if (ret) + dev_err(udma_device->dev, + "configure poe channel %u addr failed, ret = %d.\n", + id, ret); + return ret; +} + +static int config_poe_attr(struct udma_dev *udma_device, uint8_t id, bool en) +{ + struct udma_poe_cfg_attr_cmq *cmd; + struct udma_cmq_desc desc; + int ret; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_CFG_POE_ATTR, false); + cmd = (struct udma_poe_cfg_attr_cmq *)desc.data; + cmd->channel_id = cpu_to_le32(id); + cmd->rsv_en_outstd = en ? 1 : 0; + + ret = udma_cmq_send(udma_device, &desc, 1); + if (ret) + dev_err(udma_device->dev, + "configure poe channel %u attr failed, ret = %d.\n", + id, ret); + return ret; +} + +static int check_poe_channel(struct udma_dev *udma_device, uint8_t poe_ch) +{ + if (poe_ch >= udma_device->caps.poe_ch_num) { + dev_err(udma_device->dev, "invalid POE channel %u.\n", poe_ch); + return -EINVAL; + } + + return 0; +} + +int udma_user_ctl_config_poe(struct ubcore_ucontext *uctx, struct ubcore_user_ctl_in *in, + struct ubcore_user_ctl_out *out, + struct ubcore_udrv_priv *udrv_data) +{ + struct udma_poe_info poe_info; + struct udma_dev *udma_device; + int ret; + + udma_device = to_udma_dev(uctx->ub_dev); + ret = (int)copy_from_user(&poe_info, + (void *)in->addr, + sizeof(struct udma_poe_info)); + if (ret) { + dev_err(udma_device->dev, "cp from user failed in config poe, ret:%d.\n", + ret); + return -EFAULT; + } + + ret = check_poe_channel(udma_device, poe_info.poe_channel); + if (ret) { + dev_err(udma_device->dev, "check channel failed in config poe, ret:%d.\n", + ret); + return ret; + } + + ret = config_poe_attr(udma_device, poe_info.poe_channel, + !!poe_info.poe_addr); + if (ret) { + dev_err(udma_device->dev, "config attr failed in config poe, ret:%d.\n", + ret); + config_poe_addr(udma_device, poe_info.poe_channel, 0); + return ret; + } + + ret = config_poe_addr(udma_device, poe_info.poe_channel, + poe_info.poe_addr); + if (ret) + dev_err(udma_device->dev, "config addr failed in config poe, ret:%d.\n", + ret); + + return ret; +} + +static int query_poe_addr(struct udma_dev *udma_device, uint8_t id, + uint64_t *addr) +{ +#define POE_ADDR_H_SHIFT 32 + struct udma_poe_cfg_addr_cmq *resp; + struct udma_cmq_desc desc; + int ret; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_CFG_POE_ADDR, true); + resp = (struct udma_poe_cfg_addr_cmq *)desc.data; + resp->channel_id = cpu_to_le32(id); + + ret = udma_cmq_send(udma_device, &desc, 1); + if (ret) { + dev_err(udma_device->dev, + "Query poe channel %u addr failed, ret = %d.\n", + id, ret); + return ret; + } + + *addr = resp->poe_addr_l | ((uint64_t)resp->poe_addr_h << + POE_ADDR_H_SHIFT); + + return ret; +} + +static int query_poe_attr(struct udma_dev *udma_device, uint8_t id, bool *en) +{ + struct udma_poe_cfg_attr_cmq *resp; + struct udma_cmq_desc desc; + int ret; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_CFG_POE_ATTR, true); + resp = (struct udma_poe_cfg_attr_cmq *)desc.data; + resp->channel_id = cpu_to_le32(id); + + ret = udma_cmq_send(udma_device, &desc, 1); + if (ret) { + dev_err(udma_device->dev, + "Query poe channel %u attr failed, ret = %d.\n", + id, ret); + return ret; + } + + *en = !!resp->rsv_en_outstd; + + return ret; +} + +int udma_user_ctl_query_poe(struct ubcore_ucontext *uctx, struct ubcore_user_ctl_in *in, + struct ubcore_user_ctl_out *out, + struct ubcore_udrv_priv *udrv_data) +{ + struct udma_poe_info poe_info_out; + struct udma_poe_info poe_info_in; + struct udma_dev *udma_device; + uint64_t poe_addr; + bool poe_en; + int ret; + + udma_device = to_udma_dev(uctx->ub_dev); + ret = (int)copy_from_user(&poe_info_in, (void *)in->addr, + sizeof(struct udma_poe_info)); + if (ret) { + dev_err(udma_device->dev, "cp from user failed in query poe, ret:%d.\n", + ret); + return -EFAULT; + } + + ret = check_poe_channel(udma_device, poe_info_in.poe_channel); + if (ret) { + dev_err(udma_device->dev, "check channel failed in query poe, ret:%d.\n", + ret); + return ret; + } + + ret = query_poe_attr(udma_device, poe_info_in.poe_channel, &poe_en); + if (ret) { + dev_err(udma_device->dev, "query attr failed in query poe, ret:%d.\n", + ret); + return ret; + } + + ret = query_poe_addr(udma_device, poe_info_in.poe_channel, &poe_addr); + if (ret) { + dev_err(udma_device->dev, "query addr failed in query poe, ret:%d.\n", + ret); + return ret; + } + + poe_info_out.en = poe_en ? 1 : 0; + poe_info_out.poe_addr = poe_addr; + ret = (int)copy_to_user((void *)out->addr, &poe_info_out, + min(out->len, + (uint32_t)sizeof(struct udma_poe_info))); + if (ret != 0) { + dev_err(udma_device->dev, "cp to user failed in query poe, ret:%d.\n", + ret); + return -EFAULT; + } + return ret; +} + typedef int (*udma_user_ctl_opcode)(struct ubcore_ucontext *uctx, struct ubcore_user_ctl_in *in, struct ubcore_user_ctl_out *out, @@ -363,6 +562,8 @@ typedef int (*udma_user_ctl_opcode)(struct ubcore_ucontext *uctx, static udma_user_ctl_opcode g_udma_user_ctl_opcodes[] = { [UDMA_USER_CTL_FLUSH_CQE] = udma_user_ctl_flush_cqe, + [UDMA_CONFIG_POE_CHANNEL] = udma_user_ctl_config_poe, + [UDMA_QUERY_POE_CHANNEL] = udma_user_ctl_query_poe, }; int udma_user_ctl(struct ubcore_user_ctl *k_user_ctl) @@ -844,6 +1045,12 @@ static void udma_cleanup_hem(struct udma_dev *udma_dev) udma_cleanup_common_hem(udma_dev); } +void udma_set_poe_ch_num(struct udma_dev *dev) +{ +#define UDMA_POE_CH_NUM 4 + + dev->caps.poe_ch_num = UDMA_POE_CH_NUM; +} static void udma_set_devname(struct udma_dev *udma_dev, struct ubcore_device *ub_dev) @@ -938,6 +1145,7 @@ int udma_hnae_client_init(struct udma_dev *udma_dev) goto error_failed_engine_init; } + udma_set_poe_ch_num(udma_dev); ret = udma_register_device(udma_dev); if (ret) { dev_err(dev, "udma register device failed!\n"); -- Gitee From 3758e08435fb5c764351477c8b6f97e7613cbd9a Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Wed, 13 Sep 2023 15:21:34 +0800 Subject: [PATCH 37/42] hns3 udma: support config congestion control algorithms driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA --------------------------------------------------------------------- This patch supports config congestion control algorithms which as known as SCC. UDMA driver supports four scc algorithms: DCQCN, LDCP, HC3, and DIP, and each algorithm corresponds to a different QP context configuration in six fields. Only DIP need to config dip_idx. Signed-off-by: Yu Zhang --- drivers/ub/hw/hns3/hns3_udma_hw.c | 17 +++ drivers/ub/hw/hns3/hns3_udma_qp.c | 176 +++++++++++++++++++++++++++++- drivers/ub/hw/hns3/hns3_udma_qp.h | 47 ++++++++ 3 files changed, 239 insertions(+), 1 deletion(-) diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.c b/drivers/ub/hw/hns3/hns3_udma_hw.c index 24fdcf5deb2b..3fc2da50e3c4 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hw.c +++ b/drivers/ub/hw/hns3/hns3_udma_hw.c @@ -1423,6 +1423,22 @@ static void udma_free_link_table(struct udma_dev *udma_dev) free_link_table_buf(udma_dev, &priv->ext_llm); } +static void free_dip_list(struct udma_dev *udma_dev) +{ + struct udma_dip *u_dip; + struct udma_dip *tmp; + unsigned long flags; + + spin_lock_irqsave(&udma_dev->dip_list_lock, flags); + + list_for_each_entry_safe(u_dip, tmp, &udma_dev->dip_list, node) { + list_del(&u_dip->node); + kfree(u_dip); + } + + spin_unlock_irqrestore(&udma_dev->dip_list_lock, flags); +} + static int udma_get_reset_page(struct udma_dev *dev) { dev->reset_page = alloc_page(GFP_KERNEL | __GFP_ZERO); @@ -1577,6 +1593,7 @@ static void udma_hw_exit(struct udma_dev *udma_dev) udma_free_link_table(udma_dev); put_hem_table(udma_dev); + free_dip_list(udma_dev); } static int get_op_for_set_hem(uint32_t type, int step_idx, uint16_t *mbox_op, diff --git a/drivers/ub/hw/hns3/hns3_udma_qp.c b/drivers/ub/hw/hns3/hns3_udma_qp.c index 9c70394245cb..76c509ecf019 100644 --- a/drivers/ub/hw/hns3/hns3_udma_qp.c +++ b/drivers/ub/hw/hns3/hns3_udma_qp.c @@ -454,6 +454,171 @@ static int modify_qp_rtr_to_rts(struct udma_qp *qp, return 0; } +static enum udma_cong_type to_udma_cong_type(uint32_t cc_alg) +{ + switch ((enum ubcore_tp_cc_alg)cc_alg) { + case UBCORE_TP_CC_DCQCN: + return UDMA_CONG_TYPE_DCQCN; + case UBCORE_TP_CC_LDCP: + return UDMA_CONG_TYPE_LDCP; + case UBCORE_TP_CC_HC3: + return UDMA_CONG_TYPE_HC3; + case UBCORE_TP_CC_DIP: + return UDMA_CONG_TYPE_DIP; + default: + return UDMA_CONG_TYPE_DCQCN; + } +} + +static void fill_congest_type(struct udma_congestion_algorithm *congest_alg, + enum udma_cong_type qp_cong_alg) +{ + switch (qp_cong_alg) { + case UDMA_CONG_TYPE_DCQCN: + congest_alg->congest_type = CONGEST_DCQCN; + congest_alg->alg_sel = DCQCN_ALG; + congest_alg->alg_sub_sel = UNSUPPORT_CONGEST_DEGREE; + congest_alg->dip_vld = DIP_INVALID; + congest_alg->wnd_mode_sel = WND_LIMIT; + break; + case UDMA_CONG_TYPE_LDCP: + congest_alg->congest_type = CONGEST_LDCP; + congest_alg->alg_sel = WINDOW_ALG; + congest_alg->alg_sub_sel = UNSUPPORT_CONGEST_DEGREE; + congest_alg->dip_vld = DIP_INVALID; + congest_alg->wnd_mode_sel = WND_UNLIMIT; + break; + case UDMA_CONG_TYPE_HC3: + congest_alg->congest_type = CONGEST_HC3; + congest_alg->alg_sel = WINDOW_ALG; + congest_alg->alg_sub_sel = SUPPORT_CONGEST_DEGREE; + congest_alg->dip_vld = DIP_INVALID; + congest_alg->wnd_mode_sel = WND_LIMIT; + break; + case UDMA_CONG_TYPE_DIP: + congest_alg->congest_type = CONGEST_DIP; + congest_alg->alg_sel = DCQCN_ALG; + congest_alg->alg_sub_sel = UNSUPPORT_CONGEST_DEGREE; + congest_alg->dip_vld = DIP_VALID; + congest_alg->wnd_mode_sel = WND_LIMIT; + break; + default: + congest_alg->congest_type = CONGEST_DCQCN; + congest_alg->alg_sel = DCQCN_ALG; + congest_alg->alg_sub_sel = UNSUPPORT_CONGEST_DEGREE; + congest_alg->dip_vld = DIP_INVALID; + congest_alg->wnd_mode_sel = WND_LIMIT; + break; + } +} + +static int get_dip_ctx_idx(struct udma_qp *qp, + uint32_t *dip_idx) +{ + struct udma_dev *udma_dev = qp->udma_device; + uint32_t *spare_idx = udma_dev->qp_table.idx_table.spare_idx; + uint32_t *head = &udma_dev->qp_table.idx_table.head; + uint32_t *tail = &udma_dev->qp_table.idx_table.tail; + struct udma_modify_tp_attr *attr; + struct udma_dip *udma_dip; + unsigned long flags; + int ret = 0; + + attr = qp->m_attr; + spin_lock_irqsave(&udma_dev->dip_list_lock, flags); + + spare_idx[*tail] = qp->qpn; + *tail = (*tail == udma_dev->caps.num_qps - 1) ? 0 : (*tail + 1); + + list_for_each_entry(udma_dip, &udma_dev->dip_list, node) { + if (!memcmp(attr->dgid, udma_dip->dgid, UDMA_GID_SIZE)) { + *dip_idx = udma_dip->dip_idx; + goto out; + } + } + + udma_dip = kzalloc(sizeof(*udma_dip), GFP_ATOMIC); + if (!udma_dip) { + ret = -ENOMEM; + goto out; + } + + memcpy(udma_dip->dgid, attr->dgid, sizeof(attr->dgid)); + udma_dip->dip_idx = *dip_idx = spare_idx[*head]; + *head = (*head == udma_dev->caps.num_qps - 1) ? 0 : (*head + 1); + list_add_tail(&udma_dip->node, &udma_dev->dip_list); + +out: + spin_unlock_irqrestore(&udma_dev->dip_list_lock, flags); + return ret; +} + +static int udma_set_cong_fields(struct udma_qp_context *context, + struct udma_qp_context *context_mask, + struct udma_qp *qp, + const struct ubcore_tp_attr *attr) +{ + struct udma_congestion_algorithm congest_filed; + enum udma_cong_type qp_cong_alg; + uint32_t dip_idx = 0; + int ret = 0; + + qp_cong_alg = to_udma_cong_type(attr->flag.bs.cc_alg); + fill_congest_type(&congest_filed, qp_cong_alg); + + if (qp->qp_type == QPT_UD) { + congest_filed.congest_type = CONGEST_DCQCN; + congest_filed.alg_sel = DCQCN_ALG; + congest_filed.alg_sub_sel = UNSUPPORT_CONGEST_DEGREE; + congest_filed.dip_vld = DIP_INVALID; + congest_filed.wnd_mode_sel = WND_LIMIT; + } + + udma_reg_write(context, QPC_CONGEST_ALGO_TMPL_ID, + qp->udma_device->cong_algo_tmpl_id + + congest_filed.congest_type * UDMA_CONGEST_SIZE); + udma_reg_clear(context_mask, QPC_CONGEST_ALGO_TMPL_ID); + + udma_reg_write(&context->ext, QPCEX_CONGEST_ALG_SEL, congest_filed.alg_sel); + udma_reg_clear(&context_mask->ext, QPCEX_CONGEST_ALG_SEL); + + udma_reg_write(&context->ext, QPCEX_CONGEST_ALG_SUB_SEL, congest_filed.alg_sub_sel); + udma_reg_clear(&context_mask->ext, QPCEX_CONGEST_ALG_SUB_SEL); + + udma_reg_write(&context->ext, QPCEX_DIP_CTX_IDX_VLD, congest_filed.dip_vld); + udma_reg_clear(&context_mask->ext, QPCEX_DIP_CTX_IDX_VLD); + + udma_reg_write(&context->ext, QPCEX_SQ_RQ_NOT_FORBID_EN, congest_filed.wnd_mode_sel); + udma_reg_clear(&context_mask->ext, QPCEX_SQ_RQ_NOT_FORBID_EN); + + if (congest_filed.dip_vld == 0) + goto out; + + ret = get_dip_ctx_idx(qp, &dip_idx); + if (ret) { + dev_err(qp->udma_device->dev, + "failed to fill congest, ret = %d.\n", ret); + goto out; + } + qp->dip_idx = (int64_t)dip_idx; + if (dip_idx != qp->qpn) { + ret = udma_table_get(qp->udma_device, + &qp->udma_device->qp_table.sccc_table, + dip_idx); + if (ret) { + dev_err(qp->udma_device->dev, + "Failed to get SCC CTX table\n"); + goto out; + } + } + + udma_reg_write(&context->ext, QPCEX_DIP_CTX_IDX, dip_idx); + udma_reg_clear(&context_mask->ext, QPCEX_DIP_CTX_IDX); + +out: + return ret; +} + static void udma_set_spray_field(struct udma_qp *qp, const struct ubcore_tp_attr *attr, union ubcore_tp_attr_mask ubcore_mask, @@ -559,6 +724,9 @@ static void udma_set_opt_fields(struct udma_qp *qp, if (ubcore_mask.bs.flag && attr->flag.bs.oor_en && udma_dev->caps.oor_en) udma_set_oor_field(qp, attr, ubcore_mask, context, context_mask); + if (ubcore_mask.bs.flag) + udma_set_cong_fields(context, context_mask, qp, attr); + if (ubcore_mask.bs.flag && attr->flag.bs.spray_en && (udma_dev->caps.flags & UDMA_CAP_FLAG_AR)) udma_set_spray_field(qp, attr, ubcore_mask, context, context_mask); @@ -1475,9 +1643,14 @@ static void free_qpc(struct udma_dev *udma_dev, struct udma_qp *qp) if (udma_dev->caps.reorder_cq_buffer_en) udma_free_reorder_cq_buf(udma_dev, &qp->qp_attr); - if (udma_dev->caps.flags & UDMA_CAP_FLAG_QP_FLOW_CTRL) + if (udma_dev->caps.flags & UDMA_CAP_FLAG_QP_FLOW_CTRL) { udma_table_put(udma_dev, &qp_table->sccc_table, qp->qpn); + if (qp->dip_idx != qp->qpn && qp->dip_idx >= 0) + udma_table_put(udma_dev, &qp_table->sccc_table, + qp->dip_idx); + } + if (udma_dev->caps.trrl_entry_sz) udma_table_put(udma_dev, &qp_table->trrl_table, qp->qpn); @@ -1586,6 +1759,7 @@ int udma_create_qp_common(struct udma_dev *udma_dev, struct udma_qp *qp, int ret; qp->state = QPS_RESET; + qp->dip_idx = UDMA_SCC_DIP_INVALID_IDX; ret = set_qp_param(udma_dev, qp, udata, &ucmd); if (ret) { diff --git a/drivers/ub/hw/hns3/hns3_udma_qp.h b/drivers/ub/hw/hns3/hns3_udma_qp.h index 86a74e106617..cb6652ad55d7 100644 --- a/drivers/ub/hw/hns3/hns3_udma_qp.h +++ b/drivers/ub/hw/hns3/hns3_udma_qp.h @@ -125,6 +125,8 @@ struct udma_qp_context { #define QPCEX_P_TYPE_UDMA 0x1 #define MAX_SERVICE_LEVEL 0x7 +#define QPCEX_CONGEST_ALG_SEL QPCEX_FIELD_LOC(0, 0) +#define QPCEX_CONGEST_ALG_SUB_SEL QPCEX_FIELD_LOC(1, 1) #define QPCEX_DIP_CTX_IDX_VLD QPCEX_FIELD_LOC(2, 2) #define QPCEX_DIP_CTX_IDX QPCEX_FIELD_LOC(22, 3) #define QPCEX_SQ_RQ_NOT_FORBID_EN QPCEX_FIELD_LOC(23, 23) @@ -267,9 +269,54 @@ struct udma_qp { uint8_t priority; bool no_free_wqe_buf; bool force_free_wqe_buf; + int64_t dip_idx; struct udma_modify_tp_attr *m_attr; }; +struct udma_congestion_algorithm { + uint8_t congest_type; + uint8_t alg_sel; + uint8_t alg_sub_sel; + uint8_t dip_vld; + uint8_t wnd_mode_sel; +}; + +struct udma_dip { + uint8_t dgid[UDMA_GID_SIZE]; + uint32_t dip_idx; + struct list_head node; /* all dips are on a list */ +}; + +#define UDMA_CONGEST_SIZE 64 +#define UDMA_SCC_DIP_INVALID_IDX (-1) + +enum { + CONGEST_DCQCN, + CONGEST_LDCP, + CONGEST_HC3, + CONGEST_DIP, +}; + +enum { + DCQCN_ALG, + WINDOW_ALG, +}; + +enum { + UNSUPPORT_CONGEST_DEGREE, + SUPPORT_CONGEST_DEGREE, +}; + +enum { + DIP_INVALID, + DIP_VALID, +}; + +enum { + WND_LIMIT, + WND_UNLIMIT, +}; + #define gen_qpn(high, mid, low) ((high) | (mid) | (low)) int udma_modify_qp_common(struct udma_qp *qp, -- Gitee From c1ac9521e63fb41a5f086c9064564a0df2c1201d Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Wed, 13 Sep 2023 16:34:47 +0800 Subject: [PATCH 38/42] hns3 udma: support config and query parameters of SCC algorithms driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA --------------------------------------------------------------------- This patch supports config and query parameters of congestion control algorithms, which as known as SCC. UDMA driver will generates sysfs nodes corresponding to SCC parameters in /sys/class/net//device/cc_param, and read/write sysfs device attributes is equivalent to query/config parameters. Signed-off-by: Yu Zhang --- drivers/ub/hw/hns3/Makefile | 2 +- drivers/ub/hw/hns3/hns3_udma_device.h | 21 + drivers/ub/hw/hns3/hns3_udma_hw.c | 11 + drivers/ub/hw/hns3/hns3_udma_hw.h | 4 + drivers/ub/hw/hns3/hns3_udma_sysfs.c | 543 ++++++++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_sysfs.h | 173 ++++++++ 6 files changed, 753 insertions(+), 1 deletion(-) create mode 100644 drivers/ub/hw/hns3/hns3_udma_sysfs.c create mode 100644 drivers/ub/hw/hns3/hns3_udma_sysfs.h diff --git a/drivers/ub/hw/hns3/Makefile b/drivers/ub/hw/hns3/Makefile index 1d1452aad8d9..1520f7e1b9c9 100644 --- a/drivers/ub/hw/hns3/Makefile +++ b/drivers/ub/hw/hns3/Makefile @@ -11,6 +11,6 @@ $(MODULE_NAME)-objs := hns3_udma_hw.o hns3_udma_main.o hns3_udma_cmd.o \ hns3_udma_hem.o hns3_udma_qp.o hns3_udma_eq.o \ hns3_udma_db.o hns3_udma_jfc.o hns3_udma_jfr.o \ hns3_udma_segment.o hns3_udma_tp.o hns3_udma_jfs.o \ - hns3_udma_jetty.o + hns3_udma_jetty.o hns3_udma_sysfs.o obj-$(CONFIG_UB_UDMA_HNS3) := hns3_udma.o diff --git a/drivers/ub/hw/hns3/hns3_udma_device.h b/drivers/ub/hw/hns3/hns3_udma_device.h index b7ef89cb9392..a472eba1a8c2 100644 --- a/drivers/ub/hw/hns3/hns3_udma_device.h +++ b/drivers/ub/hw/hns3/hns3_udma_device.h @@ -743,6 +743,26 @@ struct udma_seg_table { struct udma_hem_table table; }; +#define UDMA_SCC_PARAM_SIZE 4 + +struct udma_scc_param { + uint32_t param[UDMA_SCC_PARAM_SIZE]; + uint32_t lifespan; + uint64_t timestamp; + enum udma_cong_type algo_type; + struct delayed_work scc_cfg_dwork; + struct udma_dev *udma_dev; + uint8_t port_num; + bool configured; +}; + +struct udma_port { + struct udma_dev *udma_dev; + uint8_t port_num; + struct kobject kobj; + struct udma_scc_param *scc_param; +}; + struct udma_dev { struct ubcore_device ub_dev; struct pci_dev *pci_dev; @@ -790,6 +810,7 @@ struct udma_dev { spinlock_t qp_list_lock; struct list_head dip_list; spinlock_t dip_list_lock; + struct udma_port port_data[UDMA_MAX_PORTS]; }; struct udma_seg { diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.c b/drivers/ub/hw/hns3/hns3_udma_hw.c index 3fc2da50e3c4..e5b588c5b6a3 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hw.c +++ b/drivers/ub/hw/hns3/hns3_udma_hw.c @@ -22,6 +22,7 @@ #include "hns3_udma_hem.h" #include "hns3_udma_eq.h" #include "hns3_udma_qp.h" +#include "hns3_udma_sysfs.h" static const struct pci_device_id udma_hw_pci_tbl[] = { { PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_UDMA_OVER_UBL), @@ -1866,7 +1867,16 @@ static int __udma_init_instance(struct hnae3_handle *handle) } handle->priv = udma_dev; + ret = udma_register_cc_sysfs(udma_dev); + if (ret) { + dev_err(udma_dev->dev, + "UDMA congest control init failed(%d)!\n", ret); + goto error_failed_cc_sysfs; + } + return 0; +error_failed_cc_sysfs: + udma_hnae_client_exit(udma_dev); error_failed_get_cfg: kfree(udma_dev->priv); error_failed_kzalloc: @@ -1883,6 +1893,7 @@ static void __udma_uninit_instance(struct hnae3_handle *handle, if (!udma_dev) return; + udma_unregister_cc_sysfs(udma_dev); handle->priv = NULL; udma_hnae_client_exit(udma_dev); diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.h b/drivers/ub/hw/hns3/hns3_udma_hw.h index e458f6778cce..9bd1ea90ab77 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hw.h +++ b/drivers/ub/hw/hns3/hns3_udma_hw.h @@ -205,6 +205,10 @@ enum udma_cmd_return_status { /* CMQ command */ enum udma_opcode_type { UDMA_QUERY_FW_VER = 0x0001, + UDMA_OPC_CFG_DCQCN_PARAM = 0x1A80, + UDMA_OPC_CFG_LDCP_PARAM = 0x1A81, + UDMA_OPC_CFG_HC3_PARAM = 0x1A82, + UDMA_OPC_CFG_DIP_PARAM = 0x1A83, UDMA_OPC_QUERY_HW_ID = 0x7032, UDMA_OPC_QUERY_HW_VER = 0x8000, UDMA_OPC_CFG_GLOBAL_PARAM = 0x8001, diff --git a/drivers/ub/hw/hns3/hns3_udma_sysfs.c b/drivers/ub/hw/hns3/hns3_udma_sysfs.c new file mode 100644 index 000000000000..0d4e3ac373e7 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_sysfs.c @@ -0,0 +1,543 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include +#include "hns3_udma_cmd.h" +#include "hns3_udma_sysfs.h" + +static enum udma_opcode_type scc_opcode[] = { + UDMA_OPC_CFG_DCQCN_PARAM, + UDMA_OPC_CFG_LDCP_PARAM, + UDMA_OPC_CFG_HC3_PARAM, + UDMA_OPC_CFG_DIP_PARAM, +}; + +static int udma_config_scc_param(struct udma_dev *udma_dev, + uint8_t port_num, enum udma_cong_type algo) +{ + struct udma_scc_param *scc_param; + struct udma_cmq_desc desc; + struct udma_port *pdata; + int ret; + + if (port_num >= udma_dev->caps.num_ports) { + dev_err_ratelimited(udma_dev->dev, + "invalid port num %u.\n", port_num); + return -ENODEV; + } + + if (algo >= UDMA_CONG_TYPE_TOTAL) { + dev_err_ratelimited(udma_dev->dev, "invalid SCC algo.\n"); + return -EINVAL; + } + + udma_cmq_setup_basic_desc(&desc, scc_opcode[algo], false); + pdata = &udma_dev->port_data[port_num]; + scc_param = &pdata->scc_param[algo]; + if (!scc_param) { + dev_err_ratelimited(udma_dev->dev, "scc_param has been freed.\n"); + return -ENODEV; + } + + memcpy(&desc.data, scc_param, sizeof(scc_param->param)); + + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) + dev_err_ratelimited(udma_dev->dev, + "failed to configure scc param, opcode: 0x%x, ret = %d.\n", + le16_to_cpu(desc.opcode), ret); + + return ret; +} + +static int udma_query_scc_param(struct udma_dev *udma_dev, + uint8_t port_num, enum udma_cong_type algo) +{ + struct udma_scc_param *scc_param; + struct udma_cmq_desc desc; + struct udma_port *pdata; + int ret; + + if (port_num >= udma_dev->caps.num_ports) { + dev_err_ratelimited(udma_dev->dev, + "invalid port num %u.\n", port_num); + return -ENODEV; + } + + if (algo >= UDMA_CONG_TYPE_TOTAL) { + dev_err_ratelimited(udma_dev->dev, "invalid SCC algo.\n"); + return -EINVAL; + } + + udma_cmq_setup_basic_desc(&desc, scc_opcode[algo], true); + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) { + dev_err_ratelimited(udma_dev->dev, + "failed to query scc param, opecode: 0x%x, ret = %d.\n", + le16_to_cpu(desc.opcode), ret); + return ret; + } + + pdata = &udma_dev->port_data[port_num]; + scc_param = &pdata->scc_param[algo]; + memcpy(scc_param, &desc.data, sizeof(scc_param->param)); + + return 0; +} + +static void scc_param_config_work(struct work_struct *work) +{ + struct udma_scc_param *scc_param = container_of(work, + struct udma_scc_param, + scc_cfg_dwork.work); + struct udma_dev *udma_dev = scc_param->udma_dev; + + udma_config_scc_param(udma_dev, scc_param->port_num, + scc_param->algo_type); +} + +static int alloc_scc_param(struct udma_dev *udma_dev, + struct udma_port *pdata) +{ + struct udma_scc_param *scc_param; + int i; + + scc_param = kcalloc(UDMA_CONG_TYPE_TOTAL, sizeof(*scc_param), + GFP_KERNEL); + if (!scc_param) + return -ENOMEM; + + for (i = 0; i < UDMA_CONG_TYPE_TOTAL; i++) { + scc_param[i].algo_type = (enum udma_cong_type)i; + scc_param[i].timestamp = jiffies; + scc_param[i].udma_dev = udma_dev; + scc_param[i].port_num = pdata->port_num; + scc_param[i].configured = false; + INIT_DELAYED_WORK(&scc_param[i].scc_cfg_dwork, + scc_param_config_work); + } + + pdata->scc_param = scc_param; + + return 0; +} + +static int scc_attr_check(struct udma_port_cc_attr *scc_attr) +{ + if (WARN_ON(scc_attr->size > sizeof(uint32_t))) + return -EINVAL; + + if (WARN_ON(scc_attr->algo_type >= UDMA_CONG_TYPE_TOTAL)) + return -EINVAL; + + return 0; +} + +static ssize_t scc_attr_show(struct udma_port *pdata, + struct udma_port_attribute *attr, char *buf) +{ + struct udma_port_cc_attr *scc_attr; + struct udma_scc_param *scc_param; + unsigned long exp_time; + uint32_t val = 0; + int ret; + + scc_attr = container_of(attr, struct udma_port_cc_attr, port_attr); + ret = scc_attr_check(scc_attr); + if (ret) + return ret; + + scc_param = &pdata->scc_param[scc_attr->algo_type]; + + /* Only HW param need be queried */ + if (scc_attr->offset < offsetof(typeof(*scc_param), lifespan)) { + exp_time = scc_param->timestamp + + msecs_to_jiffies(scc_param->lifespan); + if (time_is_before_eq_jiffies(exp_time)) { + scc_param->timestamp = jiffies; + udma_query_scc_param(pdata->udma_dev, + pdata->port_num, + scc_attr->algo_type); + } + } + + memcpy(&val, (void *)scc_param + scc_attr->offset, scc_attr->size); + + return sysfs_emit(buf, "%u\n", le32_to_cpu(val)); +} + +static ssize_t scc_attr_store(struct udma_port *pdata, + struct udma_port_attribute *attr, + const char *buf, size_t count) +{ + struct udma_port_cc_attr *scc_attr; + struct udma_scc_param *scc_param; + uint64_t lifespan_jiffies; + unsigned long exp_time; + uint32_t attr_val; + uint32_t val; + int ret; + + scc_attr = container_of(attr, struct udma_port_cc_attr, port_attr); + ret = scc_attr_check(scc_attr); + if (ret) + return ret; + + if (kstrtou32(buf, 0, &val)) + return -EINVAL; + + if (val > scc_attr->max || val < scc_attr->min) + return -EINVAL; + + attr_val = cpu_to_le32(val); + scc_param = &pdata->scc_param[scc_attr->algo_type]; + + /* get current params of this scc algo before configure it first time */ + if (scc_param->configured == false) + udma_query_scc_param(pdata->udma_dev, scc_param->port_num, + scc_param->algo_type); + + memcpy((void *)scc_param + scc_attr->offset, &attr_val, scc_attr->size); + + /* lifespan is only used for driver */ + if (scc_attr->offset >= offsetof(typeof(*scc_param), lifespan)) + return count; + + lifespan_jiffies = msecs_to_jiffies(scc_param->lifespan); + exp_time = scc_param->timestamp + lifespan_jiffies; + + if (time_is_before_eq_jiffies(exp_time)) { + scc_param->timestamp = jiffies; + queue_delayed_work(pdata->udma_dev->irq_workq, + &scc_param->scc_cfg_dwork, lifespan_jiffies); + scc_param->configured = true; + } + + return count; +} + +static umode_t scc_attr_is_visible(struct kobject *kobj, + struct attribute *attr, int i) +{ + struct udma_port_attribute *port_attr; + struct udma_port_cc_attr *scc_attr; + struct udma_dev *udma_dev; + struct udma_port *pdata; + + port_attr = container_of(attr, struct udma_port_attribute, attr); + scc_attr = container_of(port_attr, struct udma_port_cc_attr, port_attr); + pdata = container_of(kobj, struct udma_port, kobj); + udma_dev = pdata->udma_dev; + + if (!(udma_dev->caps.flags & UDMA_CAP_FLAG_QP_FLOW_CTRL)) + return 0; + + if (!(udma_dev->caps.cong_type & (1 << scc_attr->algo_type))) + return 0; + + return ATTR_RW_RONLY_RONLY; +} + +#define __UDMA_SCC_ATTR(_name, _type, _offset, _size, _min, _max) { \ + .port_attr = __ATTR(_name, 0644, scc_attr_show, scc_attr_store), \ + .algo_type = (_type), \ + .offset = (_offset), \ + .size = (_size), \ + .min = (_min), \ + .max = (_max), \ +} + +#define UDMA_DCQCN_CC_ATTR_RW(_name, NAME) \ + struct udma_port_cc_attr udma_port_attr_dcqcn_##_name = \ + __UDMA_SCC_ATTR(_name, UDMA_CONG_TYPE_DCQCN, \ + UDMA_DCQCN_##NAME##_OFS, \ + UDMA_DCQCN_##NAME##_SZ, \ + 0, UDMA_DCQCN_##NAME##_MAX) + +UDMA_DCQCN_CC_ATTR_RW(ai, AI); +UDMA_DCQCN_CC_ATTR_RW(f, F); +UDMA_DCQCN_CC_ATTR_RW(tkp, TKP); +UDMA_DCQCN_CC_ATTR_RW(tmp, TMP); +UDMA_DCQCN_CC_ATTR_RW(alp, ALP); +UDMA_DCQCN_CC_ATTR_RW(max_speed, MAX_SPEED); +UDMA_DCQCN_CC_ATTR_RW(g, G); +UDMA_DCQCN_CC_ATTR_RW(al, AL); +UDMA_DCQCN_CC_ATTR_RW(cnp_time, CNP_TIME); +UDMA_DCQCN_CC_ATTR_RW(ashift, ASHIFT); +UDMA_DCQCN_CC_ATTR_RW(lifespan, LIFESPAN); + +static struct attribute *dcqcn_param_attrs[] = { + &udma_port_attr_dcqcn_ai.port_attr.attr, + &udma_port_attr_dcqcn_f.port_attr.attr, + &udma_port_attr_dcqcn_tkp.port_attr.attr, + &udma_port_attr_dcqcn_tmp.port_attr.attr, + &udma_port_attr_dcqcn_alp.port_attr.attr, + &udma_port_attr_dcqcn_max_speed.port_attr.attr, + &udma_port_attr_dcqcn_g.port_attr.attr, + &udma_port_attr_dcqcn_al.port_attr.attr, + &udma_port_attr_dcqcn_cnp_time.port_attr.attr, + &udma_port_attr_dcqcn_ashift.port_attr.attr, + &udma_port_attr_dcqcn_lifespan.port_attr.attr, + NULL, +}; + +static const struct attribute_group dcqcn_cc_param_group = { + .name = "dcqcn_cc_param", + .attrs = dcqcn_param_attrs, + .is_visible = scc_attr_is_visible, +}; + +#define UDMA_LDCP_CC_ATTR_RW(_name, NAME) \ + struct udma_port_cc_attr udma_port_attr_ldcp_##_name = \ + __UDMA_SCC_ATTR(_name, UDMA_CONG_TYPE_LDCP, \ + UDMA_LDCP_##NAME##_OFS, \ + UDMA_LDCP_##NAME##_SZ, \ + 0, UDMA_LDCP_##NAME##_MAX) + +UDMA_LDCP_CC_ATTR_RW(cwd0, CWD0); +UDMA_LDCP_CC_ATTR_RW(alpha, ALPHA); +UDMA_LDCP_CC_ATTR_RW(gamma, GAMMA); +UDMA_LDCP_CC_ATTR_RW(beta, BETA); +UDMA_LDCP_CC_ATTR_RW(eta, ETA); +UDMA_LDCP_CC_ATTR_RW(lifespan, LIFESPAN); + +static struct attribute *ldcp_param_attrs[] = { + &udma_port_attr_ldcp_cwd0.port_attr.attr, + &udma_port_attr_ldcp_alpha.port_attr.attr, + &udma_port_attr_ldcp_gamma.port_attr.attr, + &udma_port_attr_ldcp_beta.port_attr.attr, + &udma_port_attr_ldcp_eta.port_attr.attr, + &udma_port_attr_ldcp_lifespan.port_attr.attr, + NULL, +}; + +static const struct attribute_group ldcp_cc_param_group = { + .name = "ldcp_cc_param", + .attrs = ldcp_param_attrs, + .is_visible = scc_attr_is_visible, +}; + +#define UDMA_HC3_CC_ATTR_RW(_name, NAME) \ + struct udma_port_cc_attr udma_port_attr_hc3_##_name = \ + __UDMA_SCC_ATTR(_name, UDMA_CONG_TYPE_HC3, \ + UDMA_HC3_##NAME##_OFS, \ + UDMA_HC3_##NAME##_SZ, \ + 0, UDMA_HC3_##NAME##_MAX) + +UDMA_HC3_CC_ATTR_RW(initial_window, INITIAL_WINDOW); +UDMA_HC3_CC_ATTR_RW(bandwidth, BANDWIDTH); +UDMA_HC3_CC_ATTR_RW(qlen_shift, QLEN_SHIFT); +UDMA_HC3_CC_ATTR_RW(port_usage_shift, PORT_USAGE_SHIFT); +UDMA_HC3_CC_ATTR_RW(over_period, OVER_PERIOD); +UDMA_HC3_CC_ATTR_RW(max_stage, MAX_STAGE); +UDMA_HC3_CC_ATTR_RW(gamma_shift, GAMMA_SHIFT); +UDMA_HC3_CC_ATTR_RW(lifespan, LIFESPAN); + +static struct attribute *hc3_param_attrs[] = { + &udma_port_attr_hc3_initial_window.port_attr.attr, + &udma_port_attr_hc3_bandwidth.port_attr.attr, + &udma_port_attr_hc3_qlen_shift.port_attr.attr, + &udma_port_attr_hc3_port_usage_shift.port_attr.attr, + &udma_port_attr_hc3_over_period.port_attr.attr, + &udma_port_attr_hc3_max_stage.port_attr.attr, + &udma_port_attr_hc3_gamma_shift.port_attr.attr, + &udma_port_attr_hc3_lifespan.port_attr.attr, + NULL, +}; + +static const struct attribute_group hc3_cc_param_group = { + .name = "hc3_cc_param", + .attrs = hc3_param_attrs, + .is_visible = scc_attr_is_visible, +}; + +#define UDMA_DIP_CC_ATTR_RW(_name, NAME) \ + struct udma_port_cc_attr udma_port_attr_dip_##_name = \ + __UDMA_SCC_ATTR(_name, UDMA_CONG_TYPE_DIP, \ + UDMA_DIP_##NAME##_OFS, \ + UDMA_DIP_##NAME##_SZ, \ + 0, UDMA_DIP_##NAME##_MAX) + +UDMA_DIP_CC_ATTR_RW(ai, AI); +UDMA_DIP_CC_ATTR_RW(f, F); +UDMA_DIP_CC_ATTR_RW(tkp, TKP); +UDMA_DIP_CC_ATTR_RW(tmp, TMP); +UDMA_DIP_CC_ATTR_RW(alp, ALP); +UDMA_DIP_CC_ATTR_RW(max_speed, MAX_SPEED); +UDMA_DIP_CC_ATTR_RW(g, G); +UDMA_DIP_CC_ATTR_RW(al, AL); +UDMA_DIP_CC_ATTR_RW(cnp_time, CNP_TIME); +UDMA_DIP_CC_ATTR_RW(ashift, ASHIFT); +UDMA_DIP_CC_ATTR_RW(lifespan, LIFESPAN); + +static struct attribute *dip_param_attrs[] = { + &udma_port_attr_dip_ai.port_attr.attr, + &udma_port_attr_dip_f.port_attr.attr, + &udma_port_attr_dip_tkp.port_attr.attr, + &udma_port_attr_dip_tmp.port_attr.attr, + &udma_port_attr_dip_alp.port_attr.attr, + &udma_port_attr_dip_max_speed.port_attr.attr, + &udma_port_attr_dip_g.port_attr.attr, + &udma_port_attr_dip_al.port_attr.attr, + &udma_port_attr_dip_cnp_time.port_attr.attr, + &udma_port_attr_dip_ashift.port_attr.attr, + &udma_port_attr_dip_lifespan.port_attr.attr, + NULL, +}; + +static const struct attribute_group dip_cc_param_group = { + .name = "dip_cc_param", + .attrs = dip_param_attrs, + .is_visible = scc_attr_is_visible, +}; + +const struct attribute_group *udma_attr_port_groups[] = { + &dcqcn_cc_param_group, + &ldcp_cc_param_group, + &hc3_cc_param_group, + &dip_cc_param_group, + NULL, +}; + +static ssize_t udma_port_attr_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct udma_port_attribute *port_attr; + struct udma_port *port; + + port_attr = container_of(attr, struct udma_port_attribute, attr); + port = container_of(kobj, struct udma_port, kobj); + if (!port_attr->show) + return -EIO; + + return port_attr->show(port, port_attr, buf); +} + +static ssize_t udma_port_attr_store(struct kobject *kobj, + struct attribute *attr, + const char *buf, size_t count) +{ + struct udma_port_attribute *port_attr; + struct udma_port *p; + + port_attr = container_of(attr, struct udma_port_attribute, attr); + p = container_of(kobj, struct udma_port, kobj); + if (!port_attr->store) + return -EIO; + + return port_attr->store(p, port_attr, buf, count); +} + +static void udma_port_release(struct kobject *kobj) +{ + struct udma_port *pdata = container_of(kobj, struct udma_port, kobj); + + kfree(pdata->scc_param); + pdata->scc_param = NULL; +} + +static const struct sysfs_ops udma_port_ops = { + .show = udma_port_attr_show, + .store = udma_port_attr_store, +}; + +static struct kobj_type udma_port_ktype = { + .release = udma_port_release, + .sysfs_ops = &udma_port_ops, +}; + +static int udma_register_port_sysfs(struct udma_dev *udma_dev, uint8_t port_num, + struct kobject *kobj) +{ + struct udma_port *pdata; + int ret; + + if (port_num >= udma_dev->caps.num_ports) { + dev_err(udma_dev->dev, "fail to create port sysfs for invalid port %u.\n", + port_num); + return -ENODEV; + } + + pdata = &udma_dev->port_data[port_num]; + pdata->udma_dev = udma_dev; + pdata->port_num = port_num; + ret = kobject_init_and_add(&pdata->kobj, &udma_port_ktype, + kobj, "cc_param"); + if (ret) { + dev_err(udma_dev->dev, "fail to create port(%u) sysfs, ret = %d.\n", + port_num, ret); + goto fail_kobj; + } + kobject_uevent(&pdata->kobj, KOBJ_ADD); + + ret = sysfs_create_groups(&pdata->kobj, udma_attr_port_groups); + if (ret) { + dev_err(udma_dev->dev, + "fail to create port(%u) cc param sysfs, ret = %d.\n", + port_num, ret); + goto fail_kobj; + } + + ret = alloc_scc_param(udma_dev, pdata); + if (ret) { + dev_err(udma_dev->dev, "alloc scc param failed, ret = %d!\n", + ret); + goto fail_group; + } + + return ret; + +fail_group: + sysfs_remove_groups(&pdata->kobj, udma_attr_port_groups); +fail_kobj: + kobject_put(&pdata->kobj); + return ret; +} + +static void udma_unregister_port_sysfs(struct udma_dev *udma_dev, uint8_t port_num) +{ + struct udma_port *pdata; + + pdata = &udma_dev->port_data[port_num]; + sysfs_remove_groups(&pdata->kobj, udma_attr_port_groups); + kobject_put(&pdata->kobj); +} + +int udma_register_cc_sysfs(struct udma_dev *udma_dev) +{ + uint8_t i; + uint8_t j; + + for (i = 0; i < udma_dev->caps.num_ports; i++) { + if (udma_register_port_sysfs(udma_dev, i, &udma_dev->dev->kobj) != 0) + goto err_port_attr; + } + + return 0; + +err_port_attr: + for (j = 0; j < i; j++) + udma_unregister_port_sysfs(udma_dev, j); + + return -EPERM; +} + +void udma_unregister_cc_sysfs(struct udma_dev *udma_dev) +{ + int i; + + for (i = 0; i < udma_dev->caps.num_ports; i++) + udma_unregister_port_sysfs(udma_dev, i); +} diff --git a/drivers/ub/hw/hns3/hns3_udma_sysfs.h b/drivers/ub/hw/hns3/hns3_udma_sysfs.h new file mode 100644 index 000000000000..54c3b1c5ebfd --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_sysfs.h @@ -0,0 +1,173 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_SYSFS_H +#define _UDMA_SYSFS_H + +#include +#include "hns3_udma_device.h" + +#define UDMA_DCQCN_AI_OFS 0 +#define UDMA_DCQCN_AI_SZ sizeof(u16) +#define UDMA_DCQCN_AI_MAX ((u16) (~0U)) +#define UDMA_DCQCN_F_OFS (UDMA_DCQCN_AI_OFS + UDMA_DCQCN_AI_SZ) +#define UDMA_DCQCN_F_SZ sizeof(u8) +#define UDMA_DCQCN_F_MAX ((u8) (~0U)) +#define UDMA_DCQCN_TKP_OFS (UDMA_DCQCN_F_OFS + UDMA_DCQCN_F_SZ) +#define UDMA_DCQCN_TKP_SZ sizeof(u8) +#define UDMA_DCQCN_TKP_MAX 15 +#define UDMA_DCQCN_TMP_OFS (UDMA_DCQCN_TKP_OFS + UDMA_DCQCN_TKP_SZ) +#define UDMA_DCQCN_TMP_SZ sizeof(u16) +#define UDMA_DCQCN_TMP_MAX 15 +#define UDMA_DCQCN_ALP_OFS (UDMA_DCQCN_TMP_OFS + UDMA_DCQCN_TMP_SZ) +#define UDMA_DCQCN_ALP_SZ sizeof(u16) +#define UDMA_DCQCN_ALP_MAX ((u16) (~0U)) +#define UDMA_DCQCN_MAX_SPEED_OFS (UDMA_DCQCN_ALP_OFS + \ + UDMA_DCQCN_ALP_SZ) +#define UDMA_DCQCN_MAX_SPEED_SZ sizeof(u32) +#define UDMA_DCQCN_MAX_SPEED_MAX ((u32) (~0U)) +#define UDMA_DCQCN_G_OFS (UDMA_DCQCN_MAX_SPEED_OFS + \ + UDMA_DCQCN_MAX_SPEED_SZ) +#define UDMA_DCQCN_G_SZ sizeof(u8) +#define UDMA_DCQCN_G_MAX 15 +#define UDMA_DCQCN_AL_OFS (UDMA_DCQCN_G_OFS + UDMA_DCQCN_G_SZ) +#define UDMA_DCQCN_AL_SZ sizeof(u8) +#define UDMA_DCQCN_AL_MAX ((u8) (~0U)) +#define UDMA_DCQCN_CNP_TIME_OFS (UDMA_DCQCN_AL_OFS + \ + UDMA_DCQCN_AL_SZ) +#define UDMA_DCQCN_CNP_TIME_SZ sizeof(u8) +#define UDMA_DCQCN_CNP_TIME_MAX ((u8) (~0U)) +#define UDMA_DCQCN_ASHIFT_OFS (UDMA_DCQCN_CNP_TIME_OFS + \ + UDMA_DCQCN_CNP_TIME_SZ) +#define UDMA_DCQCN_ASHIFT_SZ sizeof(u8) +#define UDMA_DCQCN_ASHIFT_MAX 15 +#define UDMA_DCQCN_LIFESPAN_OFS (UDMA_DCQCN_ASHIFT_OFS + \ + UDMA_DCQCN_ASHIFT_SZ) +#define UDMA_DCQCN_LIFESPAN_SZ sizeof(u32) +#define UDMA_DCQCN_LIFESPAN_MAX 1000 + +#define UDMA_LDCP_CWD0_OFS 0 +#define UDMA_LDCP_CWD0_SZ sizeof(u32) +#define UDMA_LDCP_CWD0_MAX ((u32) (~0U)) +#define UDMA_LDCP_ALPHA_OFS (UDMA_LDCP_CWD0_OFS + UDMA_LDCP_CWD0_SZ) +#define UDMA_LDCP_ALPHA_SZ sizeof(u8) +#define UDMA_LDCP_ALPHA_MAX ((u8) (~0U)) +#define UDMA_LDCP_GAMMA_OFS (UDMA_LDCP_ALPHA_OFS + \ + UDMA_LDCP_ALPHA_SZ) +#define UDMA_LDCP_GAMMA_SZ sizeof(u8) +#define UDMA_LDCP_GAMMA_MAX ((u8) (~0U)) +#define UDMA_LDCP_BETA_OFS (UDMA_LDCP_GAMMA_OFS + \ + UDMA_LDCP_GAMMA_SZ) +#define UDMA_LDCP_BETA_SZ sizeof(u8) +#define UDMA_LDCP_BETA_MAX ((u8) (~0U)) +#define UDMA_LDCP_ETA_OFS (UDMA_LDCP_BETA_OFS + UDMA_LDCP_BETA_SZ) +#define UDMA_LDCP_ETA_SZ sizeof(u8) +#define UDMA_LDCP_ETA_MAX ((u8) (~0U)) +#define UDMA_LDCP_LIFESPAN_OFS (4 * sizeof(u32)) +#define UDMA_LDCP_LIFESPAN_SZ sizeof(u32) +#define UDMA_LDCP_LIFESPAN_MAX 1000 + +#define UDMA_HC3_INITIAL_WINDOW_OFS 0 +#define UDMA_HC3_INITIAL_WINDOW_SZ sizeof(u32) +#define UDMA_HC3_INITIAL_WINDOW_MAX ((u32) (~0U)) +#define UDMA_HC3_BANDWIDTH_OFS (UDMA_HC3_INITIAL_WINDOW_OFS + \ + UDMA_HC3_INITIAL_WINDOW_SZ) +#define UDMA_HC3_BANDWIDTH_SZ sizeof(u32) +#define UDMA_HC3_BANDWIDTH_MAX ((u32) (~0U)) +#define UDMA_HC3_QLEN_SHIFT_OFS (UDMA_HC3_BANDWIDTH_OFS + \ + UDMA_HC3_BANDWIDTH_SZ) +#define UDMA_HC3_QLEN_SHIFT_SZ sizeof(u8) +#define UDMA_HC3_QLEN_SHIFT_MAX ((u8) (~0U)) +#define UDMA_HC3_PORT_USAGE_SHIFT_OFS (UDMA_HC3_QLEN_SHIFT_OFS + \ + UDMA_HC3_QLEN_SHIFT_SZ) +#define UDMA_HC3_PORT_USAGE_SHIFT_SZ sizeof(u8) +#define UDMA_HC3_PORT_USAGE_SHIFT_MAX ((u8) (~0U)) +#define UDMA_HC3_OVER_PERIOD_OFS (UDMA_HC3_PORT_USAGE_SHIFT_OFS + \ + UDMA_HC3_PORT_USAGE_SHIFT_SZ) +#define UDMA_HC3_OVER_PERIOD_SZ sizeof(u8) +#define UDMA_HC3_OVER_PERIOD_MAX ((u8) (~0U)) +#define UDMA_HC3_MAX_STAGE_OFS (UDMA_HC3_OVER_PERIOD_OFS + \ + UDMA_HC3_OVER_PERIOD_SZ) +#define UDMA_HC3_MAX_STAGE_SZ sizeof(u8) +#define UDMA_HC3_MAX_STAGE_MAX ((u8) (~0U)) +#define UDMA_HC3_GAMMA_SHIFT_OFS (UDMA_HC3_MAX_STAGE_OFS + \ + UDMA_HC3_MAX_STAGE_SZ) +#define UDMA_HC3_GAMMA_SHIFT_SZ sizeof(u8) +#define UDMA_HC3_GAMMA_SHIFT_MAX 15 +#define UDMA_HC3_LIFESPAN_OFS (4 * sizeof(u32)) +#define UDMA_HC3_LIFESPAN_SZ sizeof(u32) +#define UDMA_HC3_LIFESPAN_MAX 1000 + +#define UDMA_DIP_AI_OFS 0 +#define UDMA_DIP_AI_SZ sizeof(u16) +#define UDMA_DIP_AI_MAX ((u16) (~0U)) +#define UDMA_DIP_F_OFS (UDMA_DIP_AI_OFS + UDMA_DIP_AI_SZ) +#define UDMA_DIP_F_SZ sizeof(u8) +#define UDMA_DIP_F_MAX ((u8) (~0U)) +#define UDMA_DIP_TKP_OFS (UDMA_DIP_F_OFS + UDMA_DIP_F_SZ) +#define UDMA_DIP_TKP_SZ sizeof(u8) +#define UDMA_DIP_TKP_MAX 15 +#define UDMA_DIP_TMP_OFS (UDMA_DIP_TKP_OFS + UDMA_DIP_TKP_SZ) +#define UDMA_DIP_TMP_SZ sizeof(u16) +#define UDMA_DIP_TMP_MAX 15 +#define UDMA_DIP_ALP_OFS (UDMA_DIP_TMP_OFS + UDMA_DIP_TMP_SZ) +#define UDMA_DIP_ALP_SZ sizeof(u16) +#define UDMA_DIP_ALP_MAX ((u16) (~0U)) +#define UDMA_DIP_MAX_SPEED_OFS (UDMA_DIP_ALP_OFS + UDMA_DIP_ALP_SZ) +#define UDMA_DIP_MAX_SPEED_SZ sizeof(u32) +#define UDMA_DIP_MAX_SPEED_MAX ((u32) (~0U)) +#define UDMA_DIP_G_OFS (UDMA_DIP_MAX_SPEED_OFS + \ + UDMA_DIP_MAX_SPEED_SZ) +#define UDMA_DIP_G_SZ sizeof(u8) +#define UDMA_DIP_G_MAX 15 +#define UDMA_DIP_AL_OFS (UDMA_DIP_G_OFS + UDMA_DIP_G_SZ) +#define UDMA_DIP_AL_SZ sizeof(u8) +#define UDMA_DIP_AL_MAX ((u8) (~0U)) +#define UDMA_DIP_CNP_TIME_OFS (UDMA_DIP_AL_OFS + UDMA_DIP_AL_SZ) +#define UDMA_DIP_CNP_TIME_SZ sizeof(u8) +#define UDMA_DIP_CNP_TIME_MAX ((u8) (~0U)) +#define UDMA_DIP_ASHIFT_OFS (UDMA_DIP_CNP_TIME_OFS + \ + UDMA_DIP_CNP_TIME_SZ) +#define UDMA_DIP_ASHIFT_SZ sizeof(u8) +#define UDMA_DIP_ASHIFT_MAX 15 +#define UDMA_DIP_LIFESPAN_OFS (UDMA_DIP_ASHIFT_OFS + \ + UDMA_DIP_ASHIFT_SZ) +#define UDMA_DIP_LIFESPAN_SZ sizeof(u32) +#define UDMA_DIP_LIFESPAN_MAX 1000 + +#define ATTR_RW_RONLY_RONLY 0644 + +struct udma_port_attribute { + struct attribute attr; + ssize_t (*show)(struct udma_port *pdata, + struct udma_port_attribute *attr, char *buf); + ssize_t (*store)(struct udma_port *pdata, + struct udma_port_attribute *attr, const char *buf, + size_t count); +}; + +struct udma_port_cc_attr { + struct udma_port_attribute port_attr; + enum udma_cong_type algo_type; + uint32_t offset; + uint32_t size; + uint32_t min; + uint32_t max; +}; + +int udma_register_cc_sysfs(struct udma_dev *udma_dev); +void udma_unregister_cc_sysfs(struct udma_dev *udma_dev); + +#endif /* _UDMA_SYSFS_H */ -- Gitee From 1f0cdbbca09738094dc8bf6025ff8a33844212ef Mon Sep 17 00:00:00 2001 From: Weibo Zhao Date: Thu, 14 Sep 2023 14:53:05 +0800 Subject: [PATCH 39/42] hns3 udma: support of DCA driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA ------------------------------------------------------ This patch support DCA mode. DCA(dynamic context attach) is a feature that can reduce memory usage. Each TP must have a sending queue to store wqe. In sparse I/O scenarios, maintaining send queue for each TP is a waste of memory. DCA provides a way to dynamically expand or shrink the sending queue. DCA may cause a performance loss. Signed-off-by: Weibo Zhao --- drivers/ub/hw/hns3/Makefile | 2 +- drivers/ub/hw/hns3/hns3_udma_abi.h | 76 ++ drivers/ub/hw/hns3/hns3_udma_dca.c | 1235 ++++++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_dca.h | 219 +++++ drivers/ub/hw/hns3/hns3_udma_device.h | 31 + drivers/ub/hw/hns3/hns3_udma_hem.c | 9 + drivers/ub/hw/hns3/hns3_udma_hw.h | 5 + drivers/ub/hw/hns3/hns3_udma_jetty.c | 8 +- drivers/ub/hw/hns3/hns3_udma_jetty.h | 1 + drivers/ub/hw/hns3/hns3_udma_main.c | 249 ++++- drivers/ub/hw/hns3/hns3_udma_qp.c | 101 +- drivers/ub/hw/hns3/hns3_udma_qp.h | 16 + drivers/ub/hw/hns3/hns3_udma_segment.c | 1 + drivers/ub/hw/hns3/hns3_udma_tp.c | 24 +- drivers/ub/hw/hns3/hns3_udma_tp.h | 1 + 15 files changed, 1958 insertions(+), 20 deletions(-) create mode 100644 drivers/ub/hw/hns3/hns3_udma_dca.c create mode 100644 drivers/ub/hw/hns3/hns3_udma_dca.h diff --git a/drivers/ub/hw/hns3/Makefile b/drivers/ub/hw/hns3/Makefile index 1520f7e1b9c9..a9456f111843 100644 --- a/drivers/ub/hw/hns3/Makefile +++ b/drivers/ub/hw/hns3/Makefile @@ -11,6 +11,6 @@ $(MODULE_NAME)-objs := hns3_udma_hw.o hns3_udma_main.o hns3_udma_cmd.o \ hns3_udma_hem.o hns3_udma_qp.o hns3_udma_eq.o \ hns3_udma_db.o hns3_udma_jfc.o hns3_udma_jfr.o \ hns3_udma_segment.o hns3_udma_tp.o hns3_udma_jfs.o \ - hns3_udma_jetty.o hns3_udma_sysfs.o + hns3_udma_jetty.o hns3_udma_sysfs.o hns3_udma_dca.o obj-$(CONFIG_UB_UDMA_HNS3) := hns3_udma.o diff --git a/drivers/ub/hw/hns3/hns3_udma_abi.h b/drivers/ub/hw/hns3/hns3_udma_abi.h index 819aef32323c..69ecaeb402cc 100644 --- a/drivers/ub/hw/hns3/hns3_udma_abi.h +++ b/drivers/ub/hw/hns3/hns3_udma_abi.h @@ -28,11 +28,15 @@ #define UDMA_JETTY_QPN_PREFIX 0x2 #define UDMA_ADDR_4K_MASK 0xfffUL #define URMA_SEG_ACCESS_GUARD (1UL << 5) +#define UDMA_DCA_ATTACH_FLAGS_NEW_BUFFER BIT(0) +#define UDMA_DCA_INVALID_DCA_NUM ~0U enum { UDMA_MMAP_UAR_PAGE, UDMA_MMAP_DWQE_PAGE, + UDMA_MMAP_DCA_PAGE, UDMA_MMAP_RESET_PAGE, + UDMA_MMAP_TYPE_DCA }; enum udma_jfc_init_attr_mask { @@ -114,6 +118,7 @@ enum udma_qp_cap_flags { UDMA_QP_CAP_RQ_RECORD_DB = 1 << 0, UDMA_QP_CAP_SQ_RECORD_DB = 1 << 1, UDMA_QP_CAP_OWNER_DB = 1 << 2, + UDMA_QP_CAP_DYNAMIC_CTX_ATTACH = 1 << 4, UDMA_QP_CAP_DIRECT_WQE = 1 << 5, }; @@ -136,6 +141,19 @@ struct udma_create_jfs_resp { struct udma_create_tp_resp create_tp_resp; }; +struct udma_create_ctx_ucmd { + uint32_t comp; + uint32_t dca_max_qps; + uint32_t dca_unit_size; +}; + +enum udma_context_comp_mask { + UDMA_CONTEXT_MASK_DCA_PRIME_QPS = 1 << 0, + UDMA_CONTEXT_MASK_DCA_UNIT_SIZE = 1 << 1, + UDMA_CONTEXT_MASK_DCA_MAX_SIZE = 1 << 2, + UDMA_CONTEXT_MASK_DCA_MIN_SIZE = 1 << 3, +}; + struct udma_create_ctx_resp { uint32_t num_comp_vectors; uint32_t num_qps_shift; @@ -150,6 +168,9 @@ struct udma_create_ctx_resp { uint32_t max_jfs_sge; uint32_t poe_ch_num; uint64_t db_addr; + uint32_t dca_qps; + uint32_t dca_mmap_size; + uint32_t dca_mode; }; struct flush_cqe_param { @@ -163,10 +184,65 @@ struct udma_poe_info { uint64_t poe_addr; }; +struct udma_dca_reg_attr { + uintptr_t key; + uintptr_t addr; + uint32_t size; +}; + +struct udma_dca_dereg_attr { + uintptr_t free_key; + struct dca_mem *mem; +}; + +struct udma_dca_shrink_attr { + uint64_t reserved_size; +}; + +struct udma_dca_shrink_resp { + struct dca_mem *mem; + uintptr_t free_key; + uint32_t free_mems; +}; + +struct udma_dca_attach_attr { + uint64_t qpn; + uint32_t sq_offset; + uint32_t sge_offset; +}; + +struct udma_dca_attach_resp { + uint32_t alloc_flags; + uint32_t alloc_pages; + uint32_t dcan; +}; + +struct udma_dca_detach_attr { + uint64_t qpn; + uint32_t sq_idx; +}; + +struct udma_dca_query_attr { + uint64_t qpn; + uint32_t page_idx; +}; + +struct udma_dca_query_resp { + uintptr_t mem_key; + uint32_t mem_ofs; + uint32_t page_count; +}; + enum udma_user_ctl_handlers { UDMA_USER_CTL_FLUSH_CQE, UDMA_CONFIG_POE_CHANNEL, UDMA_QUERY_POE_CHANNEL, + UDMA_DCA_MEM_REG, + UDMA_DCA_MEM_DEREG, + UDMA_DCA_MEM_SHRINK, + UDMA_DCA_MEM_ATTACH, + UDMA_DCA_MEM_DETACH, + UDMA_DCA_MEM_QUERY, UDMA_OPCODE_NUM, }; diff --git a/drivers/ub/hw/hns3/hns3_udma_dca.c b/drivers/ub/hw/hns3/hns3_udma_dca.c new file mode 100644 index 000000000000..be71d355792c --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_dca.c @@ -0,0 +1,1235 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include +#include "urma/ubcore_api.h" +#include "hns3_udma_cmd.h" +#include "hns3_udma_dca.h" + +static void travel_dca_pages(struct udma_dca_ctx *ctx, void *param, + int (*cb)(struct dca_mem *, uint32_t, void *)) +{ + struct dca_mem *mem; + unsigned long flags; + uint32_t i; + bool avail; + int ret; + + spin_lock_irqsave(&ctx->pool_lock, flags); + list_for_each_entry(mem, &ctx->pool, list) { + spin_unlock_irqrestore(&ctx->pool_lock, flags); + + spin_lock(&mem->lock); + avail = dca_mem_is_available(mem); + ret = 0; + for (i = 0; avail && i < mem->page_count; i++) { + ret = cb(mem, i, param); + if (ret == DCA_MEM_STOP_ITERATE || + ret == DCA_MEM_NEXT_ITERATE) + break; + } + spin_unlock(&mem->lock); + spin_lock_irqsave(&ctx->pool_lock, flags); + + if (ret == DCA_MEM_STOP_ITERATE) + goto done; + } + +done: + spin_unlock_irqrestore(&ctx->pool_lock, flags); +} + +void udma_enable_dca(struct udma_dev *dev, struct udma_qp *qp) +{ + struct udma_dca_cfg *cfg = &qp->dca_cfg; + + spin_lock_init(&cfg->lock); + INIT_LIST_HEAD(&cfg->aging_node); + cfg->buf_id = UDMA_DCA_INVALID_BUF_ID; + cfg->npages = qp->buff_size >> UDMA_HW_PAGE_SHIFT; + cfg->dcan = UDMA_DCA_INVALID_DCA_NUM; +} + +static void stop_aging_dca_mem(struct udma_dca_ctx *ctx, + struct udma_dca_cfg *cfg, bool stop_worker) +{ + spin_lock(&ctx->aging_lock); + if (stop_worker) { + ctx->exit_aging = true; + cancel_delayed_work(&ctx->aging_dwork); + } + + spin_lock(&cfg->lock); + + if (!list_empty(&cfg->aging_node)) + list_del_init(&cfg->aging_node); + + spin_unlock(&cfg->lock); + spin_unlock(&ctx->aging_lock); +} + +static void update_dca_buf_status(struct udma_dca_ctx *ctx, uint32_t dcan, + bool en) +{ + uintptr_t *st = ctx->buf_status; + + if (st && dcan < ctx->max_qps) { + if (en) + set_bit(DCAN_TO_STAT_BIT(dcan), st); + else + clear_bit(DCAN_TO_STAT_BIT(dcan), st); + /* barrier */ + smp_mb__after_atomic(); + } +} + +static int free_buffer_pages_proc(struct dca_mem *mem, uint32_t index, + void *param) +{ + struct dca_page_free_buf_attr *attr = param; + struct dca_page_state *state; + uint32_t free_pages = 0; + bool changed = false; + bool stop = false; + uint32_t i = 0; + + for (; !stop && i < mem->page_count; i++) { + state = &mem->states[i]; + /* Change matched pages state */ + if (dca_page_is_attached(state, attr->buf_id)) { + set_dca_page_to_free(state); + changed = true; + attr->free_pages++; + if (attr->free_pages == attr->max_pages) + stop = true; + } + + if (dca_page_is_free(state)) + free_pages++; + } + + for (; changed && i < mem->page_count; i++) { + state = &mem->states[i]; + if (dca_page_is_free(state)) + free_pages++; + } + + if (changed && free_pages == mem->page_count) + attr->clean_mems++; + + return stop ? DCA_MEM_STOP_ITERATE : DCA_MEM_NEXT_ITERATE; +} + +static void free_buf_from_dca_mem(struct udma_dca_ctx *ctx, + struct udma_dca_cfg *cfg) +{ + struct dca_page_free_buf_attr attr = {}; + unsigned long flags; + uint32_t buf_id; + + update_dca_buf_status(ctx, cfg->dcan, false); + spin_lock(&cfg->lock); + buf_id = cfg->buf_id; + cfg->buf_id = UDMA_DCA_INVALID_BUF_ID; + spin_unlock(&cfg->lock); + if (buf_id == UDMA_DCA_INVALID_BUF_ID) + return; + + attr.buf_id = buf_id; + attr.max_pages = cfg->npages; + travel_dca_pages(ctx, &attr, free_buffer_pages_proc); + + /* Update free size */ + spin_lock_irqsave(&ctx->pool_lock, flags); + ctx->free_mems += attr.clean_mems; + ctx->free_size += attr.free_pages << UDMA_HW_PAGE_SHIFT; + spin_unlock_irqrestore(&ctx->pool_lock, flags); +} + +static void restart_aging_dca_mem(struct udma_dev *dev, + struct udma_dca_ctx *ctx) +{ + spin_lock(&ctx->aging_lock); + ctx->exit_aging = false; + if (!list_empty(&ctx->aging_new_list)) + queue_delayed_work(dev->irq_workq, &ctx->aging_dwork, + msecs_to_jiffies(DCA_MEM_AGEING_MSES)); + + spin_unlock(&ctx->aging_lock); +} + +static void kick_dca_buf(struct udma_dev *dev, struct udma_dca_cfg *cfg, + struct udma_dca_ctx *ctx) +{ + stop_aging_dca_mem(ctx, cfg, true); + free_buf_from_dca_mem(ctx, cfg); + restart_aging_dca_mem(dev, ctx); +} + +static void free_dca_num(struct udma_dca_cfg *cfg, struct udma_dca_ctx *ctx) +{ + if (cfg->dcan == UDMA_DCA_INVALID_DCA_NUM) + return; + + ida_free(&ctx->ida, cfg->dcan); + cfg->dcan = UDMA_DCA_INVALID_DCA_NUM; +} + +void udma_disable_dca(struct udma_dev *dev, struct udma_qp *qp) +{ + struct udma_dca_cfg *cfg = &qp->dca_cfg; + struct udma_dca_ctx *ctx = qp->dca_ctx; + + kick_dca_buf(dev, cfg, ctx); + free_dca_num(cfg, ctx); +} + +static struct dca_mem *alloc_dca_mem(struct udma_dev *dev, + struct udma_dca_ctx *ctx) +{ + struct dca_mem *mem, *found = NULL; + unsigned long flags; + + spin_lock_irqsave(&ctx->pool_lock, flags); + list_for_each_entry(mem, &ctx->pool, list) { + spin_lock(&mem->lock); + if (!mem->flags) { + found = mem; + mem->flags |= DCA_MEM_FLAGS_ALLOCED; + spin_unlock(&mem->lock); + break; + } + spin_unlock(&mem->lock); + } + spin_unlock_irqrestore(&ctx->pool_lock, flags); + + if (found) + return found; + + mem = kzalloc(sizeof(*mem), GFP_ATOMIC); + if (!mem) + return NULL; + + spin_lock_init(&mem->lock); + INIT_LIST_HEAD(&mem->list); + + mem->flags |= DCA_MEM_FLAGS_ALLOCED; + + spin_lock_irqsave(&ctx->pool_lock, flags); + list_add(&mem->list, &ctx->pool); + spin_unlock_irqrestore(&ctx->pool_lock, flags); + + return mem; +} + +static void free_dca_mem(struct dca_mem *mem) +{ + /* When iterate all DCA mems in travel_dca_pages(), we will NOT hold the + * pool's lock and just set the DCA mem as free state during the DCA is + * working until cleanup the DCA context in cleanup_dca_context(). + */ + spin_lock(&mem->lock); + mem->flags = 0; + spin_unlock(&mem->lock); +} + +static void *alloc_dca_pages(struct udma_dev *dev, struct dca_mem *mem, + struct udma_dca_reg_attr *attr) +{ + struct ubcore_device *ub_dev = &dev->ub_dev; + union ubcore_umem_flag flag = {}; + struct ubcore_umem *umem; + + flag.bs.non_pin = 0; + flag.bs.writable = 1; + umem = ubcore_umem_get(ub_dev, attr->addr, attr->size, flag); + if (IS_ERR_OR_NULL(umem)) { + dev_err(dev->dev, "failed to get uDCA pages, ret = %ld.\n", + PTR_ERR(umem)); + return NULL; + } + + mem->page_count = umem_cal_npages(umem->va, umem->length); + + return umem; +} + +static void init_dca_umem_states(struct udma_dev *dev, struct ubcore_umem *umem, + struct dca_page_state *states, uint32_t count) +{ + uint32_t npage_per_sg, k, i, total = 0; + dma_addr_t cur_addr, pre_addr = 0; + struct scatterlist *sg; + + for_each_sg(umem->sg_head.sgl, sg, umem->sg_head.nents, k) { + npage_per_sg = sg_dma_len(sg) >> UDMA_HW_PAGE_SHIFT; + for (i = 0; i < npage_per_sg; ++i) { + cur_addr = sg_dma_address(sg) + (i << UDMA_HW_PAGE_SHIFT); + if (cur_addr - pre_addr != UDMA_PAGE_SIZE) + states[total].head = 1; + + if (count <= ++total) + return; + + pre_addr = cur_addr; + } + } +} + +static struct dca_page_state *alloc_dca_states(struct udma_dev *dev, + void *pages, uint32_t count) +{ + struct dca_page_state *states; + + states = kcalloc(count, sizeof(*states), GFP_KERNEL); + if (!states) + return NULL; + + init_dca_umem_states(dev, pages, states, count); + + return states; +} + +static void stop_free_dca_buf(struct udma_dca_ctx *ctx, uint32_t dcan) +{ + uintptr_t *st = ctx->sync_status; + + if (st && dcan < ctx->max_qps) + clear_bit_unlock(DCAN_TO_SYNC_BIT(dcan), st); +} + +static uint32_t alloc_dca_num(struct udma_dca_ctx *ctx) +{ + int ret; + + ret = ida_alloc_range(&ctx->ida, 0, ctx->max_qps - 1, GFP_KERNEL); + if (ret < 0) + return UDMA_DCA_INVALID_DCA_NUM; + + stop_free_dca_buf(ctx, ret); + update_dca_buf_status(ctx, ret, false); + return ret; +} + +void udma_modify_dca(struct udma_dev *dev, struct udma_qp *qp) +{ + struct udma_dca_cfg *cfg = &qp->dca_cfg; + struct udma_dca_ctx *ctx = qp->dca_ctx; + + if (qp->state == QPS_RESET || qp->state == QPS_ERR) { + kick_dca_buf(dev, cfg, ctx); + free_dca_num(cfg, ctx); + } else if (qp->state == QPS_RTS) { + free_dca_num(cfg, ctx); + cfg->dcan = alloc_dca_num(ctx); + } +} + +int udma_register_dca_mem(struct udma_dev *dev, struct udma_ucontext *context, + struct udma_dca_reg_attr *attr) +{ + struct udma_dca_ctx *ctx = &context->dca_ctx; + struct dca_mem *mem; + unsigned long flags; + void *states; + void *pages; + + mem = alloc_dca_mem(dev, ctx); + if (!mem) { + dev_err(dev->dev, "failed to alloc dca mem.\n"); + return -ENOMEM; + } + + pages = alloc_dca_pages(dev, mem, attr); + if (!pages) { + dev_err(dev->dev, "failed to alloc dca pages.\n"); + goto err_alloc_dca_pages; + } + + states = alloc_dca_states(dev, pages, mem->page_count); + if (!states) { + dev_err(dev->dev, "failed to alloc dca states.\n"); + goto err_alloc_dca_states; + } + + spin_lock_irqsave(&ctx->pool_lock, flags); + + spin_lock(&mem->lock); + mem->pages = pages; + mem->states = states; + mem->key = attr->key; + mem->size = attr->size; + mem->flags |= DCA_MEM_FLAGS_REGISTERED; + spin_unlock(&mem->lock); + + ctx->free_mems++; + ctx->free_size += attr->size; + ctx->total_size += attr->size; + spin_unlock_irqrestore(&ctx->pool_lock, flags); + + return 0; +err_alloc_dca_states: + ubcore_umem_release(pages); +err_alloc_dca_pages: + free_dca_mem(mem); + + return -ENOMEM; +} + +void unregister_dca_mem(struct udma_dev *dev, struct udma_dca_ctx *ctx, + struct dca_mem *mem) +{ + void *states, *pages; + unsigned long flags; + + spin_lock_irqsave(&ctx->pool_lock, flags); + + spin_lock(&mem->lock); + mem->flags &= ~DCA_MEM_FLAGS_REGISTERED; + mem->page_count = 0; + pages = mem->pages; + mem->pages = NULL; + states = mem->states; + mem->states = NULL; + spin_unlock(&mem->lock); + + ctx->free_mems--; + ctx->free_size -= mem->size; + + ctx->total_size -= mem->size; + spin_unlock_irqrestore(&ctx->pool_lock, flags); + + kfree(states); + ubcore_umem_release(pages); +} + +static uint32_t get_udca_max_qps(struct udma_dev *udma_dev, + struct udma_create_ctx_ucmd *ucmd) +{ + uint32_t qp_num; + + if (ucmd->comp & UDMA_CONTEXT_MASK_DCA_PRIME_QPS) { + qp_num = ucmd->dca_max_qps; + if (!qp_num) + qp_num = udma_dev->caps.num_qps; + } else { + qp_num = 0; + } + + return qp_num; +} + +static bool start_free_dca_buf(struct udma_dca_ctx *ctx, uint32_t dcan) +{ + uintptr_t *st = ctx->sync_status; + + if (st && dcan < ctx->max_qps) + return !test_and_set_bit_lock(dcan, st); + + return true; +} + +static int udma_query_qpc(struct udma_dev *udma_dev, uint32_t qpn, + void *context) +{ + struct udma_cmd_mailbox *mailbox; + struct udma_cmq_desc desc; + struct udma_mbox *mb; + int ret; + + mailbox = udma_alloc_cmd_mailbox(udma_dev); + if (IS_ERR_OR_NULL(mailbox)) { + dev_err(udma_dev->dev, "alloc mailbox failed\n"); + ret = PTR_ERR(mailbox); + goto alloc_mailbox_fail; + } + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mb = (struct udma_mbox *)desc.data; + mbox_desc_init(mb, 0, mailbox->dma, qpn, UDMA_CMD_QUERY_QPC); + + ret = udma_cmd_mbox(udma_dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); + if (ret) { + dev_err(udma_dev->dev, "QUERY id(0x%x) cmd(0x%x) error(%d).\n", + qpn, UDMA_CMD_QUERY_QPC, ret); + goto err_mailbox; + } + memcpy(context, mailbox->buf, sizeof(struct udma_qp_context)); + +err_mailbox: + udma_free_cmd_mailbox(udma_dev, mailbox); +alloc_mailbox_fail: + return ret; +} + +static bool udma_chk_dca_buf_inactive(struct udma_dev *udma_dev, + struct udma_qp *qp) +{ + struct udma_dca_cfg *cfg = &qp->dca_cfg; + struct udma_qp_context context = {}; + uint32_t tmp, sq_idx; + int state; + int ret; + + ret = udma_query_qpc(udma_dev, qp->qpn, &context); + if (ret) { + dev_info(udma_dev->dev, "Failed to query QPC, ret = %d.\n", + ret); + return false; + } + + state = udma_reg_read(&context, QPC_QP_ST); + if (state == QP_ST_ERR || state == QP_ST_RST) + return true; + + if (qp->sq.wqe_cnt > 0) { + tmp = udma_reg_read(&context, QPC_RETRY_MSG_MSN); + sq_idx = tmp & (qp->sq.wqe_cnt - 1); + /* If SQ-PI equals to retry_msg_msn in QPC, the QP is + * inactive. + */ + if (sq_idx != cfg->sq_idx) + return false; + } + + return true; +} + +static void process_aging_dca_mem(struct udma_dev *udma_dev, + struct udma_dca_ctx *ctx) +{ + struct udma_dca_cfg *cfg, *cfg_tmp; + struct udma_qp *qp; + + spin_lock(&ctx->aging_lock); + list_for_each_entry_safe(cfg, cfg_tmp, &ctx->aging_new_list, aging_node) + list_move(&cfg->aging_node, &ctx->aging_proc_list); + + while (!ctx->exit_aging && !list_empty(&ctx->aging_proc_list)) { + cfg = list_first_entry(&ctx->aging_proc_list, + struct udma_dca_cfg, aging_node); + list_del_init_careful(&cfg->aging_node); + qp = container_of(cfg, struct udma_qp, dca_cfg); + spin_unlock(&ctx->aging_lock); + + if (start_free_dca_buf(ctx, cfg->dcan)) { + if (udma_chk_dca_buf_inactive(udma_dev, qp)) + free_buf_from_dca_mem(ctx, cfg); + + stop_free_dca_buf(ctx, cfg->dcan); + } + + spin_lock(&ctx->aging_lock); + + spin_lock(&cfg->lock); + + if (cfg->buf_id != UDMA_DCA_INVALID_BUF_ID) + list_move(&cfg->aging_node, &ctx->aging_new_list); + + spin_unlock(&cfg->lock); + } + spin_unlock(&ctx->aging_lock); +} + +static void udca_mem_aging_work(struct work_struct *work) +{ + struct udma_dca_ctx *ctx = container_of(work, struct udma_dca_ctx, + aging_dwork.work); + struct udma_ucontext *ucontext = container_of(ctx, struct udma_ucontext, + dca_ctx); + struct udma_dev *udma_dev = container_of(ucontext->uctx.ub_dev, + struct udma_dev, ub_dev); + + cancel_delayed_work(&ctx->aging_dwork); + process_aging_dca_mem(udma_dev, ctx); + if (!ctx->exit_aging) + restart_aging_dca_mem(udma_dev, ctx); +} + +static void init_dca_context(struct udma_dca_ctx *dca_ctx) +{ + INIT_LIST_HEAD(&dca_ctx->pool); + spin_lock_init(&dca_ctx->pool_lock); + dca_ctx->total_size = 0; + + ida_init(&dca_ctx->ida); + INIT_LIST_HEAD(&dca_ctx->aging_new_list); + INIT_LIST_HEAD(&dca_ctx->aging_proc_list); + spin_lock_init(&dca_ctx->aging_lock); + dca_ctx->exit_aging = false; + + INIT_DELAYED_WORK(&dca_ctx->aging_dwork, udca_mem_aging_work); +} + +static void init_udca_status(struct udma_ucontext *uctx, int udca_max_qps, + uint32_t dev_max_qps) +{ + const uint32_t bits_per_qp = 2 * UDMA_DCA_BITS_PER_STATUS; + struct udma_dca_ctx *dca_ctx = &uctx->dca_ctx; + void *kaddr; + size_t size; + + size = BITS_TO_BYTES(udca_max_qps * bits_per_qp); + dca_ctx->status_npage = DIV_ROUND_UP(size, PAGE_SIZE); + + size = dca_ctx->status_npage * PAGE_SIZE; + dca_ctx->max_qps = min_t(uint32_t, dev_max_qps, + size * BITS_PER_BYTE / bits_per_qp); + + kaddr = alloc_pages_exact(size, GFP_KERNEL | __GFP_ZERO); + if (!kaddr) + return; + + dca_ctx->buf_status = (uintptr_t *)kaddr; + dca_ctx->sync_status = (uintptr_t *)(kaddr + size / DCA_BITS_HALF); +} + +int udma_register_udca(struct udma_dev *udma_dev, + struct udma_ucontext *context, struct ubcore_udrv_priv *udrv_data) +{ + struct udma_dca_ctx *dca_ctx = &context->dca_ctx; + struct udma_create_ctx_ucmd ucmd = {}; + int max_qps; + int ret; + + ret = copy_from_user(&ucmd, (void *)udrv_data->in_addr, + min(udrv_data->in_len, (uint32_t)sizeof(ucmd))); + if (ret) { + dev_err(udma_dev->dev, "Failed to copy udata, ret = %d.\n", + ret); + return -EFAULT; + } + + if (!(udma_dev->caps.flags & UDMA_CAP_FLAG_DCA_MODE) || + ucmd.dca_unit_size == 0) + return 0; + + dca_ctx->unit_size = ucmd.dca_unit_size; + max_qps = get_udca_max_qps(udma_dev, &ucmd); + init_dca_context(dca_ctx); + if (max_qps > 0) + init_udca_status(context, max_qps, udma_dev->caps.num_qps); + + return 0; +} + +static void cleanup_dca_context(struct udma_dca_ctx *ctx) +{ + struct dca_mem *mem, *tmp; + unsigned long flags; + + spin_lock(&ctx->aging_lock); + cancel_delayed_work_sync(&ctx->aging_dwork); + spin_unlock(&ctx->aging_lock); + + spin_lock_irqsave(&ctx->pool_lock, flags); + list_for_each_entry_safe(mem, tmp, &ctx->pool, list) { + list_del(&mem->list); + spin_lock(&mem->lock); + mem->flags = 0; + spin_unlock(&mem->lock); + spin_unlock_irqrestore(&ctx->pool_lock, flags); + + kfree(mem->states); + ubcore_umem_release(mem->pages); + kfree(mem); + + spin_lock_irqsave(&ctx->pool_lock, flags); + } + ctx->total_size = 0; + spin_unlock_irqrestore(&ctx->pool_lock, flags); +} + +void udma_unregister_udca(struct udma_dev *udma_dev, + struct udma_ucontext *context) +{ + struct udma_dca_ctx *dca_ctx = &context->dca_ctx; + + if (dca_ctx->unit_size == 0) + return; + + cleanup_dca_context(dca_ctx); + if (dca_ctx->buf_status) { + free_pages_exact(dca_ctx->buf_status, + dca_ctx->status_npage * PAGE_SIZE); + dca_ctx->buf_status = NULL; + } + + ida_destroy(&dca_ctx->ida); +} + +static int dereg_dca_page_proc(struct dca_mem *mem, uint32_t index, + void *param) +{ + struct udma_dca_dereg_attr *attr = param; + + if (mem->key == attr->free_key) { + attr->mem = mem; + return DCA_MEM_STOP_ITERATE; + } else { + return DCA_MEM_NEXT_ITERATE; + } +} + +int udma_deregister_dca_mem(struct udma_dev *dev, + struct udma_ucontext *context, + struct udma_dca_dereg_attr *attr, bool from_user) +{ + if (from_user) { + travel_dca_pages(&context->dca_ctx, attr, + dereg_dca_page_proc); + if (attr->mem == NULL) { + dev_err(dev->dev, "failed to dereg DCA mems.\n"); + return -EINVAL; + } + } + + unregister_dca_mem(dev, &context->dca_ctx, attr->mem); + free_dca_mem(attr->mem); + + return 0; +} + +static int shrink_dca_page_proc(struct dca_mem *mem, uint32_t index, + void *param) +{ + struct udma_dca_shrink_resp *resp = param; + struct dca_page_state *state; + int i, free_pages; + + free_pages = 0; + for (i = 0; i < mem->page_count; i++) { + state = &mem->states[i]; + if (dca_page_is_free(state)) + free_pages++; + } + + /* No any page be used */ + if (free_pages == mem->page_count) { + /* shrink first empty DCA mem */ + if (!resp->free_mems) { + resp->mem = mem; + resp->free_key = mem->key; + } + resp->free_mems++; + } + + if (resp->free_mems > 1) + return DCA_MEM_STOP_ITERATE; + else + return DCA_MEM_NEXT_ITERATE; +} + +void udma_shrink_dca_mem(struct udma_dev *dev, struct udma_ucontext *context, + struct udma_dca_shrink_attr *attr, + struct udma_dca_shrink_resp *resp) +{ + struct udma_dca_ctx *ctx = &context->dca_ctx; + unsigned long flags; + bool need_shink; + + spin_lock_irqsave(&ctx->pool_lock, flags); + need_shink = ctx->free_mems > 0 && ctx->free_size > attr->reserved_size; + spin_unlock_irqrestore(&ctx->pool_lock, flags); + if (!need_shink) + return; + + travel_dca_pages(ctx, resp, shrink_dca_page_proc); +} + +static int query_dca_active_pages_proc(struct dca_mem *mem, uint32_t index, + void *param) +{ + struct dca_page_state *state = &mem->states[index]; + struct dca_page_query_active_attr *attr = param; + + if (!dca_page_is_active(state, attr->buf_id)) + return 0; + + if (attr->curr_index < attr->start_index) { + attr->curr_index++; + return 0; + } else if (attr->curr_index > attr->start_index) { + return DCA_MEM_STOP_ITERATE; + } + + /* Search first page in DCA mem */ + attr->page_index = index; + attr->mem_key = mem->key; + /* Search active pages in continuous addresses */ + while (index < mem->page_count) { + state = &mem->states[index]; + if (!dca_page_is_active(state, attr->buf_id)) + break; + + index++; + attr->page_count++; + } + + return DCA_MEM_STOP_ITERATE; +} + +int udma_query_dca_mem(struct udma_dev *dev, struct udma_dca_query_attr *attr, + struct udma_dca_query_resp *resp) +{ + struct dca_page_query_active_attr a_attr = {}; + struct udma_dca_ctx *ctx; + struct udma_dca_cfg *cfg; + struct udma_qp *qp; + + qp = get_qp(dev, attr->qpn); + if (qp == NULL) { + dev_err(dev->dev, "failed to find qp, qpn = 0x%llx\n", attr->qpn); + return -EINVAL; + } + cfg = &qp->dca_cfg; + ctx = qp->dca_ctx; + + a_attr.buf_id = qp->dca_cfg.buf_id; + a_attr.start_index = attr->page_idx; + travel_dca_pages(ctx, &a_attr, query_dca_active_pages_proc); + + resp->mem_key = a_attr.mem_key; + resp->mem_ofs = a_attr.page_index << UDMA_HW_PAGE_SHIFT; + resp->page_count = a_attr.page_count; + + return a_attr.page_count ? 0 : -ENOMEM; +} + +static bool dca_page_is_allocable(struct dca_page_state *state, bool head) +{ + bool is_free = dca_page_is_free(state) || dca_page_is_inactive(state); + + return head ? is_free : is_free && !state->head; +} + +static int assign_dca_pages_proc(struct dca_mem *mem, uint32_t index, + void *param) +{ + struct dca_page_assign_attr *attr = param; + struct dca_page_state *state; + uint32_t checked_pages = 0; + uint32_t start_index = 0; + uint32_t free_pages = 0; + uint32_t i = index; + + /* Check the continuous pages count is not smaller than unit count */ + for (; free_pages < attr->unit && i < mem->page_count; i++) { + checked_pages++; + state = &mem->states[i]; + if (dca_page_is_allocable(state, free_pages == 0)) { + if (free_pages == 0) + start_index = i; + + free_pages++; + } else { + free_pages = 0; + } + } + + if (free_pages < attr->unit) + return DCA_MEM_NEXT_ITERATE; + + for (i = 0; i < free_pages; i++) { + state = &mem->states[start_index + i]; + lock_dca_page_to_attach(state, attr->buf_id); + attr->total++; + } + + if (attr->total >= attr->max) + return DCA_MEM_STOP_ITERATE; + + return checked_pages; +} + +static uint32_t assign_dca_pages(struct udma_dca_ctx *ctx, uint32_t buf_id, + uint32_t count, uint32_t unit) +{ + struct dca_page_assign_attr attr = {}; + + attr.buf_id = buf_id; + attr.unit = unit; + attr.max = count; + travel_dca_pages(ctx, &attr, assign_dca_pages_proc); + return attr.total; +} + +static int clear_dca_pages_proc(struct dca_mem *mem, uint32_t index, + void *param) +{ + struct dca_page_state *state = &mem->states[index]; + struct dca_page_clear_attr *attr = param; + + if (dca_page_is_attached(state, attr->buf_id)) { + set_dca_page_to_free(state); + attr->clear_pages++; + } + + if (attr->clear_pages >= attr->max_pages) + return DCA_MEM_STOP_ITERATE; + else + return 0; +} + +static void clear_dca_pages(struct udma_dca_ctx *ctx, uint32_t buf_id, + uint32_t count) +{ + struct dca_page_clear_attr attr = {}; + + attr.buf_id = buf_id; + attr.max_pages = count; + travel_dca_pages(ctx, &attr, clear_dca_pages_proc); +} + +static uint32_t alloc_buf_from_dca_mem(struct udma_qp *qp, + struct udma_dca_ctx *ctx) +{ + uint32_t alloc_pages; + uint32_t unit_pages; + uint32_t buf_pages; + uint32_t buf_id; + + buf_pages = qp->dca_cfg.npages; + /* Gen new buf id */ + buf_id = UDMA_DCA_TO_BUF_ID(qp->qpn, qp->dca_cfg.attach_count); + + /* Assign pages from free pages */ + unit_pages = qp->mtr.hem_cfg.is_direct ? buf_pages : 1; + alloc_pages = assign_dca_pages(ctx, buf_id, buf_pages, unit_pages); + if (buf_pages != alloc_pages) { + if (alloc_pages > 0) + clear_dca_pages(ctx, buf_id, alloc_pages); + return UDMA_DCA_INVALID_BUF_ID; + } + + return buf_id; +} + +static int sync_dca_buf_offset(struct udma_dev *dev, struct udma_qp *qp, + struct udma_dca_attach_attr *attr) +{ + if (qp->sq.wqe_cnt > 0) { + if (attr->sq_offset >= qp->sge.offset) { + dev_err(dev->dev, "failed to check SQ offset = %u\n", + attr->sq_offset); + return -EINVAL; + } + qp->sq.wqe_offset = qp->sq.offset + attr->sq_offset; + } + + if (qp->sge.sge_cnt > 0) + qp->sge.wqe_offset = qp->sge.offset + attr->sge_offset; + + return 0; +} + +static int get_alloced_umem_proc(struct dca_mem *mem, uint32_t index, + void *param) +{ + struct dca_get_alloced_pages_attr *attr = param; + struct dca_page_state *states = mem->states; + struct ubcore_umem *umem = mem->pages; + uint32_t npage_per_sg, k, i; + struct scatterlist *sg; + + for_each_sg(umem->sg_head.sgl, sg, umem->sg_head.nents, k) { + npage_per_sg = sg_dma_len(sg) >> UDMA_HW_PAGE_SHIFT; + for (i = 0; i < npage_per_sg; ++i) { + if (dca_page_is_allocated(&states[i], attr->buf_id)) { + attr->pages[attr->total++] = sg_dma_address(sg) + + (i << UDMA_HW_PAGE_SHIFT); + } + if (attr->total >= attr->max) + return DCA_MEM_STOP_ITERATE; + } + } + + return DCA_MEM_NEXT_ITERATE; +} + +static int config_dca_qpc(struct udma_dev *dev, struct udma_qp *qp, + dma_addr_t *pages, uint32_t page_count) +{ + int ret; + + ret = udma_mtr_map(dev, &qp->mtr, pages, page_count); + if (ret) { + dev_err(dev->dev, "failed to map DCA pages, ret = %d.\n", ret); + return ret; + } + + ret = udma_set_dca_buf(dev, qp); + if (ret) { + dev_err(dev->dev, "failed to set DCA to HW, ret = %d.\n", ret); + return ret; + } + + return 0; +} + +static int setup_dca_buf_to_hw(struct udma_dev *dev, struct udma_qp *qp, + struct udma_dca_ctx *ctx, uint32_t buf_id, + uint32_t count) +{ + struct dca_get_alloced_pages_attr attr = {}; + dma_addr_t *pages; + int ret; + + /* alloc a tmp array to store buffer's dma address */ + pages = kvcalloc(count, sizeof(dma_addr_t), GFP_ATOMIC); + if (!pages) + return -ENOMEM; + + attr.buf_id = buf_id; + attr.pages = pages; + attr.max = count; + + travel_dca_pages(ctx, &attr, get_alloced_umem_proc); + + if (attr.total != count) { + dev_err(dev->dev, "failed to get DCA page %u != %u.\n", + attr.total, count); + ret = -ENOMEM; + goto err_get_pages; + } + + ret = config_dca_qpc(dev, qp, pages, count); +err_get_pages: + /* drop tmp array */ + kvfree(pages); + + return ret; +} + +static int active_dca_pages_proc(struct dca_mem *mem, uint32_t index, + void *param) +{ + struct dca_page_active_attr *attr = param; + struct dca_page_state *state; + bool changed = false; + bool stop = false; + int i, free_pages; + + free_pages = 0; + for (i = 0; !stop && i < mem->page_count; i++) { + state = &mem->states[i]; + if (dca_page_is_free(state)) { + free_pages++; + } else if (dca_page_is_allocated(state, attr->buf_id)) { + free_pages++; + /* Change matched pages state */ + unlock_dca_page_to_active(state, attr->buf_id); + changed = true; + attr->alloc_pages++; + if (attr->alloc_pages == attr->max_pages) + stop = true; + } + } + + for (; changed && i < mem->page_count; i++) + if (dca_page_is_free(state)) + free_pages++; + + /* Clean mem changed to dirty */ + if (changed && free_pages == mem->page_count) + attr->dirty_mems++; + + return stop ? DCA_MEM_STOP_ITERATE : DCA_MEM_NEXT_ITERATE; +} + +static uint32_t active_dca_pages(struct udma_dca_ctx *ctx, uint32_t buf_id, + uint32_t count) +{ + struct dca_page_active_attr attr = {}; + unsigned long flags; + + attr.buf_id = buf_id; + attr.max_pages = count; + travel_dca_pages(ctx, &attr, active_dca_pages_proc); + + /* Update free size */ + spin_lock_irqsave(&ctx->pool_lock, flags); + ctx->free_mems -= attr.dirty_mems; + ctx->free_size -= attr.alloc_pages << UDMA_HW_PAGE_SHIFT; + spin_unlock_irqrestore(&ctx->pool_lock, flags); + + return attr.alloc_pages; +} + +static int active_alloced_buf(struct udma_dev *dev, struct udma_dca_ctx *ctx, + struct udma_qp *qp, uint32_t buf_id, + struct udma_dca_attach_attr *attr) +{ + uint32_t active_pages; + uint32_t alloc_pages; + int ret; + + alloc_pages = qp->dca_cfg.npages; + ret = sync_dca_buf_offset(dev, qp, attr); + if (ret) { + dev_err(dev->dev, "failed to sync DCA offset, ret = %d\n", ret); + goto active_fail; + } + + ret = setup_dca_buf_to_hw(dev, qp, ctx, buf_id, alloc_pages); + if (ret) { + dev_err(dev->dev, "failed to setup DCA buf, ret = %d.\n", ret); + goto active_fail; + } + + active_pages = active_dca_pages(ctx, buf_id, alloc_pages); + if (active_pages != alloc_pages) { + dev_err(dev->dev, "failed to active DCA pages, %u != %u.\n", + active_pages, alloc_pages); + ret = -ENOBUFS; + goto active_fail; + } + + return 0; +active_fail: + clear_dca_pages(ctx, buf_id, alloc_pages); + return ret; +} + +int udma_dca_attach(struct udma_dev *dev, struct udma_dca_attach_attr *attr, + struct udma_dca_attach_resp *resp) +{ + struct udma_dca_ctx *ctx; + struct udma_dca_cfg *cfg; + struct udma_qp *qp; + uint32_t buf_id; + int ret; + + qp = get_qp(dev, attr->qpn); + if (qp == NULL) { + dev_err(dev->dev, "failed to find attach qp, qpn = 0x%llx\n", + attr->qpn); + return -EINVAL; + } + cfg = &qp->dca_cfg; + ctx = qp->dca_ctx; + + stop_aging_dca_mem(ctx, cfg, false); + resp->alloc_flags = 0; + + spin_lock(&cfg->lock); + buf_id = cfg->buf_id; + /* Already attached */ + if (buf_id != UDMA_DCA_INVALID_BUF_ID) { + resp->alloc_pages = cfg->npages; + spin_unlock(&cfg->lock); + return 0; + } + + /* Start to new attach */ + resp->alloc_pages = 0; + buf_id = alloc_buf_from_dca_mem(qp, ctx); + if (buf_id == UDMA_DCA_INVALID_BUF_ID) { + spin_unlock(&cfg->lock); + /* No report fail, need try again after the pool increased */ + return 0; + } + + ret = active_alloced_buf(dev, ctx, qp, buf_id, attr); + if (ret) { + spin_unlock(&cfg->lock); + dev_err(dev->dev, + "failed to active DCA buf for QP-%llu, ret = %d.\n", + qp->qpn, ret); + return ret; + } + + /* Attach ok */ + cfg->buf_id = buf_id; + cfg->attach_count++; + spin_unlock(&cfg->lock); + + resp->alloc_flags |= UDMA_DCA_ATTACH_FLAGS_NEW_BUFFER; + resp->alloc_pages = cfg->npages; + resp->dcan = cfg->dcan; + update_dca_buf_status(ctx, cfg->dcan, true); + + return 0; +} + +void udma_dca_disattach(struct udma_dev *dev, struct udma_dca_attach_attr *attr) +{ + struct udma_dca_ctx *ctx; + struct udma_dca_cfg *cfg; + struct udma_qp *qp; + + qp = get_qp(dev, attr->qpn); + if (qp == NULL) { + dev_err(dev->dev, "failed to find disattach qp, qpn = 0x%llx\n", + attr->qpn); + return; + } + cfg = &qp->dca_cfg; + ctx = qp->dca_ctx; + + clear_dca_pages(ctx, cfg->buf_id, qp->dca_cfg.npages); + + cfg->buf_id = UDMA_DCA_INVALID_BUF_ID; + + update_dca_buf_status(ctx, cfg->dcan, false); +} + +void udma_dca_detach(struct udma_dev *dev, struct udma_dca_detach_attr *attr) +{ + struct udma_dca_ctx *ctx; + struct udma_dca_cfg *cfg; + struct udma_qp *qp; + + qp = get_qp(dev, attr->qpn); + if (qp == NULL) { + dev_err(dev->dev, "failed to find detach qp, qpn = 0x%llx\n", + attr->qpn); + return; + } + cfg = &qp->dca_cfg; + ctx = qp->dca_ctx; + + stop_aging_dca_mem(ctx, cfg, true); + + spin_lock(&ctx->aging_lock); + spin_lock(&cfg->lock); + cfg->sq_idx = attr->sq_idx; + list_add_tail(&cfg->aging_node, &ctx->aging_new_list); + spin_unlock(&cfg->lock); + spin_unlock(&ctx->aging_lock); + + restart_aging_dca_mem(dev, ctx); +} + +static int enum_dca_pool_proc(struct dca_mem *mem, uint32_t index, void *param) +{ + struct dca_mem_enum_attr *attr = param; + int ret; + + ret = attr->enum_fn((struct dca_page_state *)(mem->states), + mem->page_count, attr->param); + + return ret ? DCA_MEM_STOP_ITERATE : DCA_MEM_NEXT_ITERATE; +} + +void udma_enum_dca_pool(struct udma_dca_ctx *dca_ctx, void *param, + udma_dca_enum_callback cb) +{ + struct dca_mem_enum_attr attr; + + attr.enum_fn = cb; + attr.param = param; + travel_dca_pages(dca_ctx, &attr, enum_dca_pool_proc); +} diff --git a/drivers/ub/hw/hns3/hns3_udma_dca.h b/drivers/ub/hw/hns3/hns3_udma_dca.h new file mode 100644 index 000000000000..09cb906c96a2 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_dca.h @@ -0,0 +1,219 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_DCA_H +#define _UDMA_DCA_H + +#include "hns3_udma_abi.h" +#include "hns3_udma_tp.h" +#include "hns3_udma_hem.h" + +#define DCA_MEM_FLAGS_ALLOCED BIT(0) +#define DCA_MEM_FLAGS_REGISTERED BIT(1) + +#define DCA_MEM_AGEING_MSES 1000 /* DCA mem ageing interval time */ +#define UDMA_DCA_INVALID_BUF_ID 0U +#define DCA_MEM_STOP_ITERATE (-1) +#define DCA_MEM_NEXT_ITERATE (-2) + +#define DCAN_TO_SYNC_BIT(n) ((n) * UDMA_DCA_BITS_PER_STATUS) +#define DCAN_TO_STAT_BIT(n) DCAN_TO_SYNC_BIT(n) + +#define UDMA_DCA_OWN_MASK GENMASK(21, 0) + +/* + * buffer id(29b) = tag(7b) + owner(22b) + * [28:22] tag : indicate the QP config update times. + * [21: 0] owner: indicate the QP to which the page belongs. + */ +#define UDMA_DCA_ID_MASK GENMASK(28, 0) +#define UDMA_DCA_TAG_MASK GENMASK(28, 22) +#define UDMA_DCA_OWN_MASK GENMASK(21, 0) + +#define UDMA_DCA_BUF_ID_TO_TAG(buf_id) (((buf_id) & UDMA_DCA_TAG_MASK) >> 22) +#define UDMA_DCA_BUF_ID_TO_QPN(buf_id) ((buf_id) & UDMA_DCA_OWN_MASK) +#define UDMA_DCA_TO_BUF_ID(qpn, tag) (((qpn) & UDMA_DCA_OWN_MASK) | \ + (((tag) << 22) & UDMA_DCA_TAG_MASK)) + +/* DCA page state (32 bit) */ +struct dca_page_state { + uint32_t buf_id : 29; /* If zero, means page can be used by any buffer. */ + uint32_t lock : 1; /* @buf_id locked this page to prepare access. */ + uint32_t active : 1; /* @buf_id is accessing this page. */ + uint32_t head : 1; /* This page is the head in a continuous address range. */ +}; + +struct dca_mem { + uint32_t flags; + struct list_head list; /* link to mem list in dca context */ + spinlock_t lock; /* protect the @flags and @list */ + uint32_t page_count; /* page count in this mem obj */ + uint64_t key; /* register by caller */ + uint32_t size; /* bytes in this mem object */ + struct dca_page_state *states; /* record each page's state */ + void *pages; /* memory handle for getting dma address */ +}; + +struct dca_page_free_buf_attr { + uint32_t buf_id; + uint32_t max_pages; + uint32_t free_pages; + uint32_t clean_mems; +}; + +struct dca_page_assign_attr { + uint32_t buf_id; + uint32_t unit; + uint32_t total; + uint32_t max; +}; + +struct dca_page_clear_attr { + uint32_t buf_id; + uint32_t max_pages; + uint32_t clear_pages; +}; + +struct dca_get_alloced_pages_attr { + uint32_t buf_id; + dma_addr_t *pages; + uint32_t total; + uint32_t max; +}; + +struct dca_page_active_attr { + uint32_t buf_id; + uint32_t max_pages; + uint32_t alloc_pages; + uint32_t dirty_mems; +}; + +struct dca_page_query_active_attr { + uint32_t buf_id; + uint32_t curr_index; + uint32_t start_index; + uint32_t page_index; + uint32_t page_count; + uint64_t mem_key; +}; + +typedef int (*udma_dca_enum_callback)(struct dca_page_state *states, + uint32_t count, void *param); + +struct dca_mem_enum_attr { + void *param; + udma_dca_enum_callback enum_fn; +}; + +static inline uint64_t umem_cal_npages(uint64_t va, uint64_t len) +{ + return (ALIGN(va + len, UDMA_PAGE_SIZE) - ALIGN_DOWN(va, UDMA_PAGE_SIZE)) / + UDMA_PAGE_SIZE; +} + +static inline bool dca_page_is_attached(struct dca_page_state *state, + uint32_t buf_id) +{ + /* only the own bit needs to be matched. */ + return (UDMA_DCA_OWN_MASK & buf_id) == + (UDMA_DCA_OWN_MASK & state->buf_id); +} + +static inline bool dca_mem_is_available(struct dca_mem *mem) +{ + return mem->flags == (DCA_MEM_FLAGS_ALLOCED | DCA_MEM_FLAGS_REGISTERED); +} + +static inline void set_dca_page_to_free(struct dca_page_state *state) +{ + state->buf_id = UDMA_DCA_INVALID_BUF_ID; + state->active = 0; + state->lock = 0; +} + +static inline bool dca_page_is_free(struct dca_page_state *state) +{ + return state->buf_id == UDMA_DCA_INVALID_BUF_ID; +} + +static inline bool dca_page_is_active(struct dca_page_state *state, + uint32_t buf_id) +{ + /* all buf id bits must be matched */ + return (UDMA_DCA_ID_MASK & buf_id) == state->buf_id && + !state->lock && state->active; +} + +static inline bool dca_page_is_inactive(struct dca_page_state *state) +{ + return !state->lock && !state->active; +} + +static inline void lock_dca_page_to_attach(struct dca_page_state *state, + uint32_t buf_id) +{ + state->buf_id = UDMA_DCA_ID_MASK & buf_id; + state->active = 0; + state->lock = 1; +} + +static inline bool dca_page_is_allocated(struct dca_page_state *state, + uint32_t buf_id) +{ + return dca_page_is_attached(state, buf_id) && state->lock; +} + +static inline void unlock_dca_page_to_active(struct dca_page_state *state, + uint32_t buf_id) +{ + state->buf_id = UDMA_DCA_ID_MASK & buf_id; + state->active = 1; + state->lock = 0; +} + +void udma_enable_dca(struct udma_dev *dev, struct udma_qp *qp); +void udma_disable_dca(struct udma_dev *dev, struct udma_qp *qp); + +void udma_modify_dca(struct udma_dev *dev, struct udma_qp *qp); + +int udma_register_dca_mem(struct udma_dev *dev, struct udma_ucontext *context, + struct udma_dca_reg_attr *attr); +void unregister_dca_mem(struct udma_dev *dev, struct udma_dca_ctx *ctx, + struct dca_mem *mem); +int udma_deregister_dca_mem(struct udma_dev *dev, + struct udma_ucontext *context, + struct udma_dca_dereg_attr *attr, bool from_user); + +void udma_shrink_dca_mem(struct udma_dev *dev, struct udma_ucontext *context, + struct udma_dca_shrink_attr *attr, + struct udma_dca_shrink_resp *resp); + +int udma_query_dca_mem(struct udma_dev *dev, struct udma_dca_query_attr *attr, + struct udma_dca_query_resp *resp); + +int udma_dca_attach(struct udma_dev *dev, struct udma_dca_attach_attr *attr, + struct udma_dca_attach_resp *resp); +void udma_dca_disattach(struct udma_dev *dev, struct udma_dca_attach_attr *attr); +void udma_dca_detach(struct udma_dev *dev, struct udma_dca_detach_attr *attr); + +int udma_register_udca(struct udma_dev *udma_dev, + struct udma_ucontext *context, struct ubcore_udrv_priv *udrv_data); + +void udma_unregister_udca(struct udma_dev *udma_dev, + struct udma_ucontext *context); + +void udma_enum_dca_pool(struct udma_dca_ctx *dca_ctx, void *param, + udma_dca_enum_callback cb); +#endif diff --git a/drivers/ub/hw/hns3/hns3_udma_device.h b/drivers/ub/hw/hns3/hns3_udma_device.h index a472eba1a8c2..e080a4b1630c 100644 --- a/drivers/ub/hw/hns3/hns3_udma_device.h +++ b/drivers/ub/hw/hns3/hns3_udma_device.h @@ -144,6 +144,9 @@ #define UDMA_MIN_JFS_DEPTH 64 +#define UDMA_DCA_BITS_PER_STATUS 1 +#define DCA_BITS_HALF 2 + enum { NO_ARMED = 0x0 }; @@ -300,6 +303,8 @@ struct udma_buf_attr { } region[UDMA_MAX_BT_REGION]; uint32_t region_count; /* valid region count */ uint32_t page_shift; /* buffer page shift */ + /* only alloc buffer-required MTT memory */ + bool mtt_only; }; struct udma_buf_list { @@ -389,10 +394,36 @@ struct udma_db { void *virt_addr; }; +struct udma_dca_ctx { + struct list_head pool; /* all DCA mems link to @pool */ + spinlock_t pool_lock; /* protect @pool */ + uint32_t free_mems; /* free mem num in pool */ + size_t free_size; /* free mem size in pool */ + size_t total_size; /* total size in pool */ + size_t max_size; /* max size the pool can expand to */ + size_t min_size; /* shrink if @free_size > @min_size */ + uint32_t unit_size; /* unit size per DCA mem */ + + uint32_t max_qps; + uint32_t status_npage; + struct ida ida; + + uintptr_t *buf_status; + uintptr_t *sync_status; + + bool exit_aging; + struct list_head aging_proc_list; + struct list_head aging_new_list; + spinlock_t aging_lock; + struct delayed_work aging_dwork; +}; + struct udma_ucontext { struct ubcore_ucontext uctx; struct udma_uar uar; uint64_t pdn; + struct udma_dca_ctx dca_ctx; + void *dca_dbgfs; }; struct udma_cmd_context { diff --git a/drivers/ub/hw/hns3/hns3_udma_hem.c b/drivers/ub/hw/hns3/hns3_udma_hem.c index 7950c82306bb..e425efaa258f 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hem.c +++ b/drivers/ub/hw/hns3/hns3_udma_hem.c @@ -1920,6 +1920,15 @@ int udma_mtr_create(struct udma_dev *udma_dev, struct udma_mtr *mtr, return ret; } + /* The caller has its own buffer list and invokes the udma_mtr_map() + * to finish the MTT configuration. + */ + if (buf_attr->mtt_only) { + mtr->umem = NULL; + mtr->kmem = NULL; + return 0; + } + ret = mtr_alloc_bufs(udma_dev, mtr, buf_attr, user_addr, is_user); if (ret) { dev_err(dev, "failed to alloc mtr bufs, ret = %d.\n", ret); diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.h b/drivers/ub/hw/hns3/hns3_udma_hw.h index 9bd1ea90ab77..6c6bdc372a30 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hw.h +++ b/drivers/ub/hw/hns3/hns3_udma_hw.h @@ -202,6 +202,11 @@ enum udma_cmd_return_status { CMD_NOT_EXIST = 2, }; +enum { + QP_ST_RST = 0, + QP_ST_ERR = 6, +}; + /* CMQ command */ enum udma_opcode_type { UDMA_QUERY_FW_VER = 0x0001, diff --git a/drivers/ub/hw/hns3/hns3_udma_jetty.c b/drivers/ub/hw/hns3/hns3_udma_jetty.c index d3c31fbccf20..2684a9a554c0 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jetty.c +++ b/drivers/ub/hw/hns3/hns3_udma_jetty.c @@ -199,6 +199,7 @@ static int set_jetty_buf_attr(struct udma_dev *udma_dev, return -EINVAL; buf_attr->region_count = idx; + buf_attr->mtt_only = false; buf_attr->page_shift = PAGE_SHIFT + udma_dev->caps.mtt_buf_pg_sz; return 0; @@ -235,6 +236,11 @@ static int alloc_jetty_buf(struct udma_dev *dev, struct udma_jetty *jetty, xa_init(&jetty->srm_node_table); } else if (cfg->trans_mode == UBCORE_TP_RC) { jetty->rc_node.buf_addr = ucmd.buf_addr; + if (!ucmd.buf_addr) { + jetty->dca_en = true; + return 0; + } + ret = set_jetty_buf_attr(dev, jetty, &buf_attr); if (ret) { dev_err(dev->dev, @@ -355,7 +361,7 @@ int free_jetty_buf(struct udma_dev *dev, struct udma_jetty *jetty) jetty->qp.qpn); udma_destroy_qp_common(dev, &jetty->qp); - } else if (jetty->tp_mode == UBCORE_TP_RC) { + } else if (jetty->tp_mode == UBCORE_TP_RC && !jetty->dca_en) { udma_db_unmap_user(dev, &jetty->rc_node.sdb); udma_mtr_destroy(dev, &jetty->rc_node.mtr); } diff --git a/drivers/ub/hw/hns3/hns3_udma_jetty.h b/drivers/ub/hw/hns3/hns3_udma_jetty.h index 1fe237fbc57c..25c8b6e3bc54 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jetty.h +++ b/drivers/ub/hw/hns3/hns3_udma_jetty.h @@ -45,6 +45,7 @@ struct udma_jetty { struct udma_qpn_bitmap qpn_map; uint32_t jetty_id; struct mutex tp_mutex; + bool dca_en; }; static inline struct udma_jetty *to_udma_jetty(struct ubcore_jetty *ubcore_jetty) diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c index a29680132fa6..53f5d237f66b 100644 --- a/drivers/ub/hw/hns3/hns3_udma_main.c +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -26,6 +26,7 @@ #include "hns3_udma_jfs.h" #include "hns3_udma_segment.h" #include "hns3_udma_jetty.h" +#include "hns3_udma_dca.h" #include "hns3_udma_cmd.h" static int udma_set_eid(struct ubcore_device *dev, union ubcore_eid eid) @@ -69,7 +70,8 @@ static int udma_uar_alloc(struct udma_dev *udma_dev, struct udma_uar *uar) return 0; } -static int udma_init_ctx_resp(struct udma_dev *dev, struct ubcore_udrv_priv *udrv_data) +static int udma_init_ctx_resp(struct udma_dev *dev, struct ubcore_udrv_priv *udrv_data, + struct udma_dca_ctx *dca_ctx) { struct udma_create_ctx_resp resp = {}; int ret; @@ -89,6 +91,12 @@ static int udma_init_ctx_resp(struct udma_dev *dev, struct ubcore_udrv_priv *udr resp.db_addr = pci_resource_start(dev->pci_dev, UDMA_DEV_START_OFFSET) + UDMA_DB_ADDR_OFFSET; + if (dev->caps.flags & UDMA_CAP_FLAG_DCA_MODE) { + resp.dca_qps = dca_ctx->max_qps; + resp.dca_mmap_size = dca_ctx->status_npage * PAGE_SIZE; + resp.dca_mode = dev->caps.flags & UDMA_CAP_FLAG_DCA_MODE; + } + ret = copy_to_user((void *)udrv_data->out_addr, &resp, min(udrv_data->out_len, (uint32_t)sizeof(resp))); if (ret) @@ -122,8 +130,14 @@ static struct ubcore_ucontext *udma_alloc_ucontext(struct ubcore_device *dev, goto err_alloc_ucontext; } + ret = udma_register_udca(udma_dev, context, udrv_data); + if (ret) { + dev_err(udma_dev->dev, "Register udca Failed.\n"); + goto err_alloc_uar; + } + context->pdn = uasid; /* Use the UASID as pd number */ - ret = udma_init_ctx_resp(udma_dev, udrv_data); + ret = udma_init_ctx_resp(udma_dev, udrv_data, &context->dca_ctx); if (ret) { dev_err(udma_dev->dev, "Init ctx resp failed.\n"); goto err_alloc_uar; @@ -143,6 +157,9 @@ static int udma_free_ucontext(struct ubcore_ucontext *uctx) struct udma_ucontext *context = to_udma_ucontext(uctx); struct udma_dev *udma_dev = to_udma_dev(uctx->ub_dev); + if (udma_dev->caps.flags & UDMA_CAP_FLAG_DCA_MODE) + udma_unregister_udca(udma_dev, context); + ida_free(&udma_dev->uar_ida.ida, (int)context->uar.logic_idx); kfree(context); return 0; @@ -158,6 +175,38 @@ static uint64_t get_mmap_idx(struct vm_area_struct *vma) return ((vma->vm_pgoff >> MAP_INDEX_SHIFT) & MAP_INDEX_MASK); } +static int mmap_dca(struct ubcore_ucontext *context, struct vm_area_struct *vma) +{ + struct udma_ucontext *uctx = to_udma_ucontext(context); + struct udma_dca_ctx *ctx = &uctx->dca_ctx; + struct page **pages; + unsigned long num; + int ret; + + if (vma->vm_end - vma->vm_start != (ctx->status_npage * PAGE_SIZE)) + return -EINVAL; + + if (!(vma->vm_flags & VM_WRITE) || + !(vma->vm_flags & VM_SHARED) || + (vma->vm_flags & VM_EXEC)) + return -EPERM; + + if (!ctx->buf_status) + return -EOPNOTSUPP; + + pages = kcalloc(ctx->status_npage, sizeof(struct page *), GFP_KERNEL); + if (!pages) + return -ENOMEM; + + for (num = 0; num < ctx->status_npage; num++) + pages[num] = virt_to_page(ctx->buf_status + num * PAGE_SIZE); + + ret = vm_insert_pages(vma, vma->vm_start, pages, &num); + kfree(pages); + + return ret; +} + static int udma_mmap(struct ubcore_ucontext *uctx, struct vm_area_struct *vma) { struct udma_dev *udma_dev = to_udma_dev(uctx->ub_dev); @@ -198,6 +247,10 @@ static int udma_mmap(struct ubcore_ucontext *uctx, struct vm_area_struct *vma) PAGE_SIZE, vma->vm_page_prot)) return -EAGAIN; break; + case UDMA_MMAP_TYPE_DCA: + if (mmap_dca(uctx, vma)) + return -EAGAIN; + break; default: dev_err(udma_dev->dev, "mmap failed, cmd(%d) not support\n", cmd); @@ -555,6 +608,192 @@ int udma_user_ctl_query_poe(struct ubcore_ucontext *uctx, struct ubcore_user_ctl return ret; } +int udma_user_ctl_dca_reg(struct ubcore_ucontext *uctx, struct ubcore_user_ctl_in *in, + struct ubcore_user_ctl_out *out, struct ubcore_udrv_priv *udrv_data) +{ + struct udma_dev *udma_device = to_udma_dev(uctx->ub_dev); + struct udma_ucontext *context = to_udma_ucontext(uctx); + struct udma_dca_reg_attr attr = {}; + int ret; + + ret = (int)copy_from_user(&attr, (void *)in->addr, + sizeof(struct udma_dca_reg_attr)); + if (ret) { + dev_err(udma_device->dev, "cp from user failed in dca reg, ret:%d.\n", + ret); + return -EFAULT; + } + + ret = udma_register_dca_mem(udma_device, context, &attr); + if (ret) { + dev_err(udma_device->dev, + "register dca mem failed, ret:%d.\n", ret); + return ret; + } + + return 0; +} + +int udma_user_ctl_dca_dereg(struct ubcore_ucontext *uctx, struct ubcore_user_ctl_in *in, + struct ubcore_user_ctl_out *out, + struct ubcore_udrv_priv *udrv_data) +{ + struct udma_dev *udma_device = to_udma_dev(uctx->ub_dev); + struct udma_ucontext *context = to_udma_ucontext(uctx); + struct udma_dca_dereg_attr attr = {}; + int ret; + + ret = (int)copy_from_user(&attr, (void *)in->addr, + sizeof(struct udma_dca_dereg_attr)); + if (ret) { + dev_err(udma_device->dev, "cp from user failed in dca dereg, ret:%d.\n", + ret); + return -EFAULT; + } + + attr.mem = NULL; + ret = udma_deregister_dca_mem(udma_device, context, &attr, true); + if (ret) { + dev_err(udma_device->dev, "deregister dca mem failed, ret:%d.\n", ret); + return -EFAULT; + } + + return 0; +} + +int udma_user_ctl_dca_shrink(struct ubcore_ucontext *uctx, struct ubcore_user_ctl_in *in, + struct ubcore_user_ctl_out *out, + struct ubcore_udrv_priv *udrv_data) +{ + struct udma_dev *udma_device = to_udma_dev(uctx->ub_dev); + struct udma_ucontext *context = to_udma_ucontext(uctx); + struct udma_dca_shrink_attr shrink_attr = {}; + struct udma_dca_shrink_resp shrink_resp = {}; + struct udma_dca_dereg_attr dereg_attr = {}; + int ret; + + ret = (int)copy_from_user(&shrink_attr, (void *)in->addr, + sizeof(struct udma_dca_shrink_attr)); + if (ret) { + dev_err(udma_device->dev, "cp from user failed in dca shrink, ret:%d.\n", + ret); + return -EFAULT; + } + + udma_shrink_dca_mem(udma_device, context, &shrink_attr, &shrink_resp); + + if (shrink_resp.free_mems >= 1) { + dereg_attr.mem = shrink_resp.mem; + udma_deregister_dca_mem(udma_device, context, &dereg_attr, false); + shrink_resp.mem = NULL; + } + + ret = (int)copy_to_user((void *)out->addr, &shrink_resp, + min(out->len, + (uint32_t)sizeof(struct udma_dca_shrink_resp))); + if (ret) { + dev_err(udma_device->dev, "cp to user failed in dca_shrink, ret:%d.\n", + ret); + return -EFAULT; + } + + return 0; +} + +int udma_user_ctl_dca_attach(struct ubcore_ucontext *uctx, struct ubcore_user_ctl_in *in, + struct ubcore_user_ctl_out *out, + struct ubcore_udrv_priv *udrv_data) +{ + struct udma_dev *udma_device = to_udma_dev(uctx->ub_dev); + struct udma_dca_attach_attr attr = {}; + struct udma_dca_attach_resp resp = {}; + int ret; + + ret = (int)copy_from_user(&attr, (void *)in->addr, + sizeof(struct udma_dca_attach_attr)); + if (ret) { + dev_err(udma_device->dev, "cp from user failed in dca attach, ret:%d.\n", + ret); + return -EFAULT; + } + + ret = udma_dca_attach(udma_device, &attr, &resp); + if (ret) { + dev_err(udma_device->dev, "attach dca mem failed, ret:%d.\n", + ret); + return ret; + } + + ret = (int)copy_to_user((void *)out->addr, &resp, + min(out->len, + (uint32_t)sizeof(struct udma_dca_attach_resp))); + if (ret) { + udma_dca_disattach(udma_device, &attr); + dev_err(udma_device->dev, "copy_to_user failed in dca_attach, ret:%d.\n", + ret); + return -EFAULT; + } + + return 0; +} + +int udma_user_ctl_dca_detach(struct ubcore_ucontext *uctx, struct ubcore_user_ctl_in *in, + struct ubcore_user_ctl_out *out, + struct ubcore_udrv_priv *udrv_data) +{ + struct udma_dev *udma_device = to_udma_dev(uctx->ub_dev); + struct udma_dca_detach_attr attr = {}; + int ret; + + ret = (int)copy_from_user(&attr, (void *)in->addr, + sizeof(struct udma_dca_detach_attr)); + if (ret) { + dev_err(udma_device->dev, "cp from user failed in dca detach, ret:%d.\n", + ret); + return -EFAULT; + } + + udma_dca_detach(udma_device, &attr); + + return 0; +} + +int udma_user_ctl_dca_query(struct ubcore_ucontext *uctx, struct ubcore_user_ctl_in *in, + struct ubcore_user_ctl_out *out, + struct ubcore_udrv_priv *udrv_data) +{ + struct udma_dev *udma_device = to_udma_dev(uctx->ub_dev); + struct udma_dca_query_attr attr = {}; + struct udma_dca_query_resp resp = {}; + int ret; + + ret = (int)copy_from_user(&attr, (void *)in->addr, + sizeof(struct udma_dca_query_attr)); + if (ret) { + dev_err(udma_device->dev, "copy_from_user failed in dca query, ret:%d.\n", + ret); + return -EFAULT; + } + + ret = udma_query_dca_mem(udma_device, &attr, &resp); + if (ret) { + dev_err(udma_device->dev, "query dca mem failed, ret:%d.\n", + ret); + return ret; + } + + ret = (int)copy_to_user((void *)out->addr, &resp, + min(out->len, + (uint32_t)sizeof(struct udma_dca_query_resp))); + if (ret) { + dev_err(udma_device->dev, "cp to user failed in dca_query, ret:%d.\n", + ret); + return -EFAULT; + } + + return 0; +} + typedef int (*udma_user_ctl_opcode)(struct ubcore_ucontext *uctx, struct ubcore_user_ctl_in *in, struct ubcore_user_ctl_out *out, @@ -564,6 +803,12 @@ static udma_user_ctl_opcode g_udma_user_ctl_opcodes[] = { [UDMA_USER_CTL_FLUSH_CQE] = udma_user_ctl_flush_cqe, [UDMA_CONFIG_POE_CHANNEL] = udma_user_ctl_config_poe, [UDMA_QUERY_POE_CHANNEL] = udma_user_ctl_query_poe, + [UDMA_DCA_MEM_REG] = udma_user_ctl_dca_reg, + [UDMA_DCA_MEM_DEREG] = udma_user_ctl_dca_dereg, + [UDMA_DCA_MEM_SHRINK] = udma_user_ctl_dca_shrink, + [UDMA_DCA_MEM_ATTACH] = udma_user_ctl_dca_attach, + [UDMA_DCA_MEM_DETACH] = udma_user_ctl_dca_detach, + [UDMA_DCA_MEM_QUERY] = udma_user_ctl_dca_query, }; int udma_user_ctl(struct ubcore_user_ctl *k_user_ctl) diff --git a/drivers/ub/hw/hns3/hns3_udma_qp.c b/drivers/ub/hw/hns3/hns3_udma_qp.c index 76c509ecf019..025d5ee5efc5 100644 --- a/drivers/ub/hw/hns3/hns3_udma_qp.c +++ b/drivers/ub/hw/hns3/hns3_udma_qp.c @@ -16,6 +16,7 @@ #include #include #include "hns3_udma_abi.h" +#include "hns3_udma_dca.h" #include "hns3_udma_jfs.h" #include "hns3_udma_jfr.h" #include "hns3_udma_jfc.h" @@ -176,6 +177,30 @@ static int udma_pass_qpc_to_hw(struct udma_dev *udma_device, return ret; } +int udma_set_dca_buf(struct udma_dev *dev, struct udma_qp *qp) +{ + struct udma_qp_context ctx[2] = {}; + struct udma_qp_context *msk = ctx + 1; + struct udma_qp_context *qpc = ctx; + int ret; + + memset(msk, 0xff, dev->caps.qpc_sz); + + ret = config_qp_sq_buf(dev, qp, qpc, msk); + if (ret) { + dev_err(dev->dev, "failed to config sq qpc, ret = %d.\n", ret); + return ret; + } + + ret = udma_pass_qpc_to_hw(dev, qpc, msk, qp); + if (ret) { + dev_err(dev->dev, "failed to modify DCA buf, ret = %d.\n", ret); + return ret; + } + + return 0; +} + static bool check_qp_timeout_cfg_range(struct udma_dev *udma_device, const uint8_t *timeout) { @@ -449,6 +474,11 @@ static int modify_qp_rtr_to_rts(struct udma_qp *qp, udma_reg_clear(context_mask, QPC_TX_CQN); } + if (qp->en_flags & UDMA_QP_CAP_DYNAMIC_CTX_ATTACH) { + udma_reg_enable(context, QPC_DCA_MODE); + udma_reg_clear(context_mask, QPC_DCA_MODE); + } + set_qpc_wqe_cnt(qp, context, context_mask); return 0; @@ -890,6 +920,10 @@ int udma_modify_qp_common(struct udma_qp *qp, qp->state = new_state; + if (qp->qp_type == QPT_RC && + qp->en_flags & UDMA_QP_CAP_DYNAMIC_CTX_ATTACH) + udma_modify_dca(udma_device, qp); + out: return ret; } @@ -1334,7 +1368,7 @@ void clean_jetty_x_qpn_bitmap(struct udma_qpn_bitmap *qpn_map) } static int set_wqe_buf_attr(struct udma_dev *udma_dev, struct udma_qp *qp, - struct udma_buf_attr *buf_attr) + struct udma_buf_attr *buf_attr, bool dca_en) { uint32_t idx = 0; int buf_size; @@ -1368,43 +1402,76 @@ static int set_wqe_buf_attr(struct udma_dev *udma_dev, struct udma_qp *qp, return -EINVAL; buf_attr->region_count = idx; - buf_attr->page_shift = UDMA_HW_PAGE_SHIFT + udma_dev->caps.mtt_buf_pg_sz; + + if (dca_en) { + /* When enable DCA, there's no need to alloc buffer now, and + * the page shift should be fixed to 4K. + */ + buf_attr->mtt_only = true; + buf_attr->page_shift = UDMA_HW_PAGE_SHIFT; + } else { + buf_attr->mtt_only = false; + buf_attr->page_shift = UDMA_HW_PAGE_SHIFT + + udma_dev->caps.mtt_buf_pg_sz; + } return 0; } static int alloc_wqe_buf(struct udma_dev *dev, struct udma_qp *qp, - struct udma_buf_attr *buf_attr, uint64_t addr) + struct udma_buf_attr *buf_attr, uint64_t addr, + bool dca_en) { int ret; - if ((PAGE_SIZE <= UDMA_DWQE_SIZE) && - (dev->caps.flags & UDMA_CAP_FLAG_DIRECT_WQE) && - (qp->qpn < UDMA_DWQE_MMAP_QP_NUM)) + if (dca_en) { + /* DCA must be enabled after the buffer attr is configured. */ + udma_enable_dca(dev, qp); + qp->en_flags |= UDMA_QP_CAP_DYNAMIC_CTX_ATTACH; + } else if ((PAGE_SIZE <= UDMA_DWQE_SIZE) && + (dev->caps.flags & UDMA_CAP_FLAG_DIRECT_WQE) && + (qp->qpn < UDMA_DWQE_MMAP_QP_NUM)) { qp->en_flags |= UDMA_QP_CAP_DIRECT_WQE; + } ret = udma_mtr_create(dev, &qp->mtr, buf_attr, PAGE_SHIFT + dev->caps.mtt_ba_pg_sz, addr, true); - if (ret) + if (ret) { dev_err(dev->dev, "failed to create WQE mtr, ret = %d.\n", ret); + if (dca_en) + udma_disable_dca(dev, qp); + } return ret; } +static bool check_dca_is_enable(struct udma_dev *udma_dev, struct udma_qp *qp, + uint64_t buf_addr) +{ + if (qp->qp_type != QPT_RC || + !(udma_dev->caps.flags & UDMA_CAP_FLAG_DCA_MODE)) + return false; + + /* If the user QP's buffer addr is 0, the DCA mode should be enabled */ + return !buf_addr; +} + static int alloc_qp_wqe(struct udma_dev *udma_dev, struct udma_qp *qp, uint64_t buf_addr) { struct device *dev = udma_dev->dev; struct udma_buf_attr buf_attr = {}; + bool dca_en; int ret; - ret = set_wqe_buf_attr(udma_dev, qp, &buf_attr); + dca_en = check_dca_is_enable(udma_dev, qp, buf_addr); + ret = set_wqe_buf_attr(udma_dev, qp, &buf_attr, dca_en); if (ret) { dev_err(dev, "failed to set WQE attr, ret = %d.\n", ret); return ret; } - ret = alloc_wqe_buf(udma_dev, qp, &buf_attr, buf_addr); + ret = alloc_wqe_buf(udma_dev, qp, &buf_attr, buf_addr, dca_en); if (ret) { dev_err(dev, "failed to alloc WQE buf, ret = %d.\n", ret); return ret; @@ -1659,7 +1726,9 @@ static void free_qpc(struct udma_dev *udma_dev, struct udma_qp *qp) static void free_qp_db(struct udma_dev *udma_dev, struct udma_qp *qp) { - if (is_rc_jetty(&qp->qp_attr) || qp->no_free_wqe_buf) + if ((is_rc_jetty(&qp->qp_attr) && + !(qp->en_flags & UDMA_QP_CAP_DYNAMIC_CTX_ATTACH)) || + qp->no_free_wqe_buf) return; if (qp->en_flags & UDMA_QP_CAP_SQ_RECORD_DB) @@ -1668,10 +1737,14 @@ static void free_qp_db(struct udma_dev *udma_dev, struct udma_qp *qp) static void free_wqe_buf(struct udma_dev *dev, struct udma_qp *qp) { - if (is_rc_jetty(&qp->qp_attr) || qp->no_free_wqe_buf) + if ((is_rc_jetty(&qp->qp_attr) && + !(qp->en_flags & UDMA_QP_CAP_DYNAMIC_CTX_ATTACH)) || + qp->no_free_wqe_buf) return; udma_mtr_destroy(dev, &qp->mtr); + if (qp->en_flags & UDMA_QP_CAP_DYNAMIC_CTX_ATTACH) + udma_disable_dca(dev, qp); } static void free_qp_wqe(struct udma_dev *udma_dev, struct udma_qp *qp) @@ -1751,15 +1824,18 @@ static uint32_t udma_get_jetty_qpn(struct udma_qp *qp) int udma_create_qp_common(struct udma_dev *udma_dev, struct udma_qp *qp, struct ubcore_udata *udata) { + struct udma_ucontext *uctx = to_udma_ucontext(udata->uctx); struct udma_qp_attr *qp_attr = &qp->qp_attr; struct udma_qp_context ctx[2] = {0}; struct device *dev = udma_dev->dev; struct udma_create_tp_ucmd ucmd; struct udma_create_tp_resp resp; + bool udma_alloc_sq_flag = false; int ret; qp->state = QPS_RESET; qp->dip_idx = UDMA_SCC_DIP_INVALID_IDX; + qp->dca_ctx = udata->uctx ? &uctx->dca_ctx : qp->dca_ctx; ret = set_qp_param(udma_dev, qp, udata, &ucmd); if (ret) { @@ -1779,7 +1855,8 @@ int udma_create_qp_common(struct udma_dev *udma_dev, struct udma_qp *qp, } if (udma_qp_need_alloc_sq(qp_attr)) { - if (is_rc_jetty(qp_attr)) { + udma_alloc_sq_flag = is_rc_jetty(qp_attr) && (ucmd.buf_addr || qp_attr->is_tgt); + if (udma_alloc_sq_flag) { qp->mtr = qp_attr->jetty->rc_node.mtr; qp->sdb = qp_attr->jetty->rc_node.sdb; qp->en_flags |= UDMA_QP_CAP_SQ_RECORD_DB; diff --git a/drivers/ub/hw/hns3/hns3_udma_qp.h b/drivers/ub/hw/hns3/hns3_udma_qp.h index cb6652ad55d7..24a1d8a2e50a 100644 --- a/drivers/ub/hw/hns3/hns3_udma_qp.h +++ b/drivers/ub/hw/hns3/hns3_udma_qp.h @@ -100,6 +100,7 @@ struct udma_qp_context { #define QPC_SQ_CUR_SGE_BLK_ADDR_L QPC_FIELD_LOC(1439, 1408) #define QPC_SQ_CUR_SGE_BLK_ADDR_H QPC_FIELD_LOC(1459, 1440) #define QPC_OWNER_MODE QPC_FIELD_LOC(1536, 1536) +#define QPC_DCA_MODE QPC_FIELD_LOC(1542, 1542) #define QPC_SQ_MAX_PSN QPC_FIELD_LOC(1567, 1544) #define QPC_RMT_E2E QPC_FIELD_LOC(1660, 1660) #define QPC_RETRY_NUM_INIT QPC_FIELD_LOC(1690, 1688) @@ -235,6 +236,18 @@ struct udma_qp_sge { int wqe_offset; }; +struct udma_dca_cfg { + spinlock_t lock; + uint32_t attach_count; + uint32_t buf_id; + uint32_t dcan; + void **buf_list; + uint32_t npages; + uint32_t sq_idx; + bool aging_enable; + struct list_head aging_node; +}; + struct udma_qp { struct udma_dev *udma_device; enum udma_qp_type qp_type; @@ -246,6 +259,8 @@ struct udma_qp { struct udma_jfc *recv_jfc; uint64_t en_flags; struct udma_mtr mtr; + struct udma_dca_cfg dca_cfg; + struct udma_dca_ctx *dca_ctx; uint32_t buff_size; enum udma_qp_state state; void (*event)(struct udma_qp *qp, @@ -338,5 +353,6 @@ int udma_flush_cqe(struct udma_dev *udma_dev, struct udma_qp *udma_qp, uint32_t sq_pi); void udma_qp_event(struct udma_dev *udma_dev, uint32_t qpn, int event_type); void copy_send_jfc(struct udma_qp *from_qp, struct udma_qp *to_qp); +int udma_set_dca_buf(struct udma_dev *dev, struct udma_qp *qp); #endif /* _UDMA_QP_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_segment.c b/drivers/ub/hw/hns3/hns3_udma_segment.c index 159c5a2bd99b..acb9c2ee9715 100644 --- a/drivers/ub/hw/hns3/hns3_udma_segment.c +++ b/drivers/ub/hw/hns3/hns3_udma_segment.c @@ -98,6 +98,7 @@ static int alloc_seg_pbl(struct udma_dev *udma_dev, struct udma_seg *seg, buf_attr.region[0].size = seg->size; buf_attr.region[0].hopnum = seg->pbl_hop_num; buf_attr.region_count = 1; + buf_attr.mtt_only = false; err = udma_mtr_create(udma_dev, &seg->pbl_mtr, &buf_attr, udma_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT, diff --git a/drivers/ub/hw/hns3/hns3_udma_tp.c b/drivers/ub/hw/hns3/hns3_udma_tp.c index 8178eb1ea44c..9f118710779d 100644 --- a/drivers/ub/hw/hns3/hns3_udma_tp.c +++ b/drivers/ub/hw/hns3/hns3_udma_tp.c @@ -19,7 +19,12 @@ #include "hns3_udma_jfs.h" #include "hns3_udma_jetty.h" #include "hns3_udma_hem.h" +#include "hns3_udma_dca.h" #include "hns3_udma_tp.h" +struct udma_qp *get_qp(struct udma_dev *udma_device, uint32_t qpn) +{ + return (struct udma_qp *)xa_load(&udma_device->qp_table.xa, qpn); +} static enum udma_qp_state to_udma_qp_state(enum ubcore_tp_state state) { @@ -229,7 +234,8 @@ static void udma_set_tp(struct ubcore_device *dev, const struct ubcore_tp_cfg *c tp->ubcore_tp.state = UBCORE_TP_STATE_RESET; } -static void copy_attr_to_pre_tp(struct udma_qp *from_qp, struct udma_qp *to_qp) +static void copy_attr_to_pre_tp(struct udma_dev *udma_device, bool is_rm, + struct udma_qp *from_qp, struct udma_qp *to_qp) { if (from_qp->qp_attr.is_tgt) return; @@ -238,11 +244,15 @@ static void copy_attr_to_pre_tp(struct udma_qp *from_qp, struct udma_qp *to_qp) to_qp->sq = from_qp->sq; to_qp->sdb = from_qp->sdb; to_qp->priority = from_qp->priority; + to_qp->dca_ctx = from_qp->dca_ctx; to_qp->en_flags = from_qp->en_flags; to_qp->buff_size = from_qp->buff_size; + if (to_qp->en_flags & UDMA_QP_CAP_DYNAMIC_CTX_ATTACH) + udma_enable_dca(udma_device, to_qp); udma_mtr_move(&from_qp->mtr, &to_qp->mtr); - copy_send_jfc(from_qp, to_qp); + if (is_rm) + copy_send_jfc(from_qp, to_qp); to_qp->qp_attr.cap.max_send_wr = from_qp->qp_attr.cap.max_send_wr; to_qp->qp_attr.cap.max_send_sge = from_qp->qp_attr.cap.max_send_sge; @@ -266,7 +276,8 @@ static int udma_store_jetty_tp(struct udma_dev *udma_device, if (jetty->tp_mode == UBCORE_TP_RM) { pre_tp = (struct udma_tp *)xa_load(&jetty->srm_node_table, hash); if (pre_tp) { - copy_attr_to_pre_tp(&tp->qp, &pre_tp->qp); + copy_attr_to_pre_tp(udma_device, true, &tp->qp, + &pre_tp->qp); *fail_ret_tp = &pre_tp->ubcore_tp; return 0; } @@ -284,7 +295,12 @@ static int udma_store_jetty_tp(struct udma_dev *udma_device, } else { tjetty_hash = udma_get_jetty_hash(&jetty->rc_node.tjetty_id); - if (tjetty_hash == hash) { + if (tjetty_hash == hash && + (tp->qp.en_flags & UDMA_QP_CAP_DYNAMIC_CTX_ATTACH)) { + copy_attr_to_pre_tp(udma_device, false, &tp->qp, + &jetty->rc_node.tp->qp); + *fail_ret_tp = &jetty->rc_node.tp->ubcore_tp; + } else if (tjetty_hash == hash) { *fail_ret_tp = &jetty->rc_node.tp->ubcore_tp; } else { dev_err(udma_device->dev, diff --git a/drivers/ub/hw/hns3/hns3_udma_tp.h b/drivers/ub/hw/hns3/hns3_udma_tp.h index 673f2b7fe600..84781db93ad7 100644 --- a/drivers/ub/hw/hns3/hns3_udma_tp.h +++ b/drivers/ub/hw/hns3/hns3_udma_tp.h @@ -43,5 +43,6 @@ struct ubcore_tp *udma_create_tp(struct ubcore_device *dev, int udma_destroy_tp(struct ubcore_tp *tp); int udma_modify_tp(struct ubcore_tp *tp, const struct ubcore_tp_attr *attr, union ubcore_tp_attr_mask mask); +struct udma_qp *get_qp(struct udma_dev *udma_device, uint32_t qpn); #endif /* _UDMA_TP_H */ -- Gitee From d90f5613c15afe13bac220356e1ccc51b59d9dd6 Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Thu, 14 Sep 2023 19:25:22 +0800 Subject: [PATCH 40/42] hns3 udma: support config params of multipath function for UM Jetty/JFS driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA --------------------------------------------------------------------- This patch supports config params of multipath function for UM Jetty/JFS in kernel mode by module param. These params are contained in the struct udma_create_tp_resp and pass to user space by copy_to_user, then different source port numbers be set for each UM sq wqe in user mode. Signed-off-by: Yu Zhang --- drivers/ub/hw/hns3/hns3_udma_abi.h | 7 +++++++ drivers/ub/hw/hns3/hns3_udma_qp.c | 21 +++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/drivers/ub/hw/hns3/hns3_udma_abi.h b/drivers/ub/hw/hns3/hns3_udma_abi.h index 69ecaeb402cc..29b66ba20179 100644 --- a/drivers/ub/hw/hns3/hns3_udma_abi.h +++ b/drivers/ub/hw/hns3/hns3_udma_abi.h @@ -122,11 +122,18 @@ enum udma_qp_cap_flags { UDMA_QP_CAP_DIRECT_WQE = 1 << 5, }; +struct udp_srcport { + bool um_spray_en; + uint16_t um_data_udp_start; + uint8_t um_udp_range; +}; + struct udma_create_tp_resp { uint64_t cap_flags; uint32_t qpn; uint32_t path_mtu; uint8_t priority; + struct udp_srcport um_srcport; }; struct udma_create_jetty_resp { diff --git a/drivers/ub/hw/hns3/hns3_udma_qp.c b/drivers/ub/hw/hns3/hns3_udma_qp.c index 025d5ee5efc5..7b98f7188d83 100644 --- a/drivers/ub/hw/hns3/hns3_udma_qp.c +++ b/drivers/ub/hw/hns3/hns3_udma_qp.c @@ -15,6 +15,7 @@ #include #include +#include #include "hns3_udma_abi.h" #include "hns3_udma_dca.h" #include "hns3_udma_jfs.h" @@ -27,6 +28,10 @@ #include "hns3_udma_db.h" #include "hns3_udma_qp.h" +static bool um_spray_en; +static ushort um_data_udp_start; +static ushort um_udp_range; + static void set_qpc_wqe_cnt(struct udma_qp *qp, struct udma_qp_context *context, struct udma_qp_context *context_mask) @@ -1900,6 +1905,10 @@ int udma_create_qp_common(struct udma_dev *udma_dev, struct udma_qp *qp, resp.qpn = udma_get_jetty_qpn(qp); resp.path_mtu = udma_dev->caps.max_mtu; + resp.um_srcport.um_spray_en = um_spray_en; + resp.um_srcport.um_data_udp_start = (uint16_t)um_data_udp_start; + resp.um_srcport.um_udp_range = (uint8_t)um_udp_range + + UDP_RANGE_BASE; ret = copy_to_user((void *)udata->udrv_data->out_addr, &resp, min(udata->udrv_data->out_len, (uint32_t)sizeof(resp))); @@ -2065,3 +2074,15 @@ void udma_qp_event(struct udma_dev *udma_dev, uint32_t qpn, int event_type) if (refcount_dec_and_test(&qp->refcount)) complete(&qp->free); } + +module_param(um_spray_en, bool, 0644); +MODULE_PARM_DESC(um_spray_en, + "Set whether to enable the multipath function for UM Jetty/Jfs, default: 0(0:off, 1:on)"); + +module_param(um_data_udp_start, ushort, 0644); +MODULE_PARM_DESC(um_data_udp_start, + "Set the Initial source port number for UM Jetty/Jfs, valid when um_spray_en is set 1"); + +module_param(um_udp_range, ushort, 0644); +MODULE_PARM_DESC(um_udp_range, + "Set the variable bits of source port number for UM Jetty/Jfs, valid when um_spray_en is set 1, range:0-8, default: 0. 0 ~ (7 + um_udp_range) bits of source port are variable"); -- Gitee From b21b2454c8f32120815f6e3d798e4b60d1cb4e47 Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Tue, 19 Sep 2023 20:40:40 +0800 Subject: [PATCH 41/42] hns3 udma: support dfx function of query resources driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA --------------------------------------------------------------------- This patch supports dfx function of query resources as follows: 1. query resources by write ID of resources to sysfs node: TP context, JFC context, JFR context, and SEG entry. 2. query resources by URMA: ID list of each resources, details of TP, JFS, JFR, JFC, Jetty, and SEG. Signed-off-by: Yu Zhang --- drivers/ub/hw/hns3/Makefile | 3 +- drivers/ub/hw/hns3/hns3_udma_device.h | 7 + drivers/ub/hw/hns3/hns3_udma_dfx.c | 1161 ++++++++++++++++++++++++ drivers/ub/hw/hns3/hns3_udma_dfx.h | 139 +++ drivers/ub/hw/hns3/hns3_udma_eq.c | 2 + drivers/ub/hw/hns3/hns3_udma_hw.c | 23 + drivers/ub/hw/hns3/hns3_udma_hw.h | 2 + drivers/ub/hw/hns3/hns3_udma_jetty.c | 87 ++ drivers/ub/hw/hns3/hns3_udma_jfc.c | 71 ++ drivers/ub/hw/hns3/hns3_udma_jfr.c | 76 ++ drivers/ub/hw/hns3/hns3_udma_jfs.c | 78 ++ drivers/ub/hw/hns3/hns3_udma_main.c | 2 + drivers/ub/hw/hns3/hns3_udma_segment.c | 76 ++ drivers/ub/hw/hns3/hns3_udma_tp.c | 71 ++ 14 files changed, 1797 insertions(+), 1 deletion(-) create mode 100644 drivers/ub/hw/hns3/hns3_udma_dfx.c create mode 100644 drivers/ub/hw/hns3/hns3_udma_dfx.h diff --git a/drivers/ub/hw/hns3/Makefile b/drivers/ub/hw/hns3/Makefile index a9456f111843..533cadbd8263 100644 --- a/drivers/ub/hw/hns3/Makefile +++ b/drivers/ub/hw/hns3/Makefile @@ -11,6 +11,7 @@ $(MODULE_NAME)-objs := hns3_udma_hw.o hns3_udma_main.o hns3_udma_cmd.o \ hns3_udma_hem.o hns3_udma_qp.o hns3_udma_eq.o \ hns3_udma_db.o hns3_udma_jfc.o hns3_udma_jfr.o \ hns3_udma_segment.o hns3_udma_tp.o hns3_udma_jfs.o \ - hns3_udma_jetty.o hns3_udma_sysfs.o hns3_udma_dca.o + hns3_udma_jetty.o hns3_udma_sysfs.o hns3_udma_dca.o \ + hns3_udma_dfx.o obj-$(CONFIG_UB_UDMA_HNS3) := hns3_udma.o diff --git a/drivers/ub/hw/hns3/hns3_udma_device.h b/drivers/ub/hw/hns3/hns3_udma_device.h index e080a4b1630c..78e03558fe75 100644 --- a/drivers/ub/hw/hns3/hns3_udma_device.h +++ b/drivers/ub/hw/hns3/hns3_udma_device.h @@ -250,6 +250,12 @@ enum udma_qp_state { QPS_ERR = 6, }; +enum udma_eq_dfx { + UDMA_DFX_AEQE, + UDMA_DFX_CEQE, + UDMA_DFX_EQ_TOTAL +}; + enum { UDMA_BUF_DIRECT = BIT(0), UDMA_BUF_NOSLEEP = BIT(1), @@ -837,6 +843,7 @@ struct udma_dev { struct udma_hem_table cqc_timer_table; struct udma_hem_table gmv_table; uint64_t dwqe_page; + uint64_t dfx_cnt[UDMA_DFX_EQ_TOTAL]; struct list_head qp_list; spinlock_t qp_list_lock; struct list_head dip_list; diff --git a/drivers/ub/hw/hns3/hns3_udma_dfx.c b/drivers/ub/hw/hns3/hns3_udma_dfx.c new file mode 100644 index 000000000000..c7437290ecb0 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_dfx.c @@ -0,0 +1,1161 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include +#include "hns3_udma_jfr.h" +#include "hns3_udma_cmd.h" +#include "hns3_udma_tp.h" +#include "hns3_udma_jfc.h" +#include "hns3_udma_segment.h" +#include "hns3_udma_dfx.h" + +struct class *drv_class; +struct device *drv_device; +static int major; +static int udma_dev_count; +struct udma_dfx_dev g_udma_dfx_list[MAX_UDMA_DEV] = {NULL}; +const struct file_operations chr_ops = { + .owner = THIS_MODULE, +}; + +static int udma_dfx_read_buf(char *str, const char *buf) +{ + const char *str_buf = buf; + int blk_cnt = 0; + int cnt = 0; + + while (*str_buf == ' ') { + str_buf++; + blk_cnt++; + } + while (str_buf[cnt] != ' ' && str_buf[cnt] != '\0' && + cnt < UDMA_DFX_STR_LEN_MAX - 1) + cnt++; + + if (((uint32_t)(blk_cnt + cnt) < strlen(buf)) || str_buf[cnt] != '\0') + return -EINVAL; + + memcpy(str, str_buf, cnt); + str[cnt] = '\0'; + + return 0; +} + +static int udma_dfx_query_context(struct udma_dev *udma_dev, uint32_t id, + void *context, uint32_t len, uint16_t op) +{ + struct udma_cmd_mailbox *mailbox; + struct udma_cmq_desc desc; + struct udma_mbox *mb; + int ret; + + mailbox = udma_alloc_cmd_mailbox(udma_dev); + if (IS_ERR(mailbox)) { + dev_err(udma_dev->dev, "alloc mailbox failed\n"); + ret = PTR_ERR(mailbox); + goto alloc_mailbox_fail; + } + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mb = (struct udma_mbox *)desc.data; + mbox_desc_init(mb, 0, mailbox->dma, id, op); + + ret = udma_cmd_mbox(udma_dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); + if (ret) { + dev_err(udma_dev->dev, "QUERY id(0x%x) cmd(0x%x) error(%d).\n", + id, op, ret); + goto err_mailbox; + } + memcpy(context, mailbox->buf, len); + +err_mailbox: + udma_free_cmd_mailbox(udma_dev, mailbox); +alloc_mailbox_fail: + return ret; +} + +static void udma_dfx_seg_print(struct udma_dev *udma_dev, uint32_t seg_key, + struct udma_mpt_entry *mpt_entry) +{ + uint32_t *mpt = (uint32_t *)mpt_entry; + uint32_t i; + + dev_info(udma_dev->dev, + "************ SEG/MPT(0x%8x) ENTRY INFO *************\n", + seg_key); + for (i = 0; i < (sizeof(*mpt_entry) / sizeof(uint32_t)); i++) { + pr_info("MPT(byte%4lu): %08x\n", (i + 1) * sizeof(uint32_t), *mpt); + mpt++; + } + dev_info(udma_dev->dev, + "*********************************************************\n"); +} + +static int udma_dfx_seg_store(const char *p_buf, struct udma_dfx_info *udma_dfx) +{ + struct udma_dev *udma_dev = (struct udma_dev *)udma_dfx->priv; + struct udma_mpt_entry mpt_entry; + char str[UDMA_DFX_STR_LEN_MAX]; + uint32_t mpt_index; + uint32_t seg_key; + int ret; + + ret = udma_dfx_read_buf(str, p_buf); + if (ret) { + dev_info(udma_dev->dev, "the inputing is invalid\n"); + return ret; + } + + if (kstrtouint(str, 0, &seg_key)) { + dev_err(udma_dev->dev, "convert str failed\n"); + return -EINVAL; + } + + mpt_index = key_to_hw_index(seg_key) & (udma_dev->caps.num_mtpts - 1); + ret = udma_dfx_query_context(udma_dev, mpt_index, &mpt_entry, + sizeof(mpt_entry), UDMA_CMD_QUERY_MPT); + if (ret) { + dev_err(udma_dev->dev, "query seg context failed, ret = %d\n", + ret); + return ret; + } + + udma_dfx_seg_print(udma_dev, seg_key, &mpt_entry); + return ret; +} + +static void udma_dfx_qpc_print(struct udma_dev *udma_dev, uint32_t qpn, + struct udma_qp_context *qp_context) +{ + uint32_t *qpc = (uint32_t *)qp_context; + uint32_t i; + + dev_info(udma_dev->dev, + "************ TP/QP(0x%8x) CONTEXT INFO *************\n", + qpn); + for (i = 0; i < (sizeof(*qp_context) / sizeof(uint32_t)); i++) { + pr_info("QPC(byte%4lu): %08x\n", (i + 1) * sizeof(uint32_t), *qpc); + qpc++; + } + dev_info(udma_dev->dev, + "*********************************************************\n"); +} + +static int udma_dfx_tp_store(const char *p_buf, struct udma_dfx_info *udma_dfx) +{ + struct udma_dev *udma_dev = (struct udma_dev *)udma_dfx->priv; + struct udma_qp_context qp_context; + char str[UDMA_DFX_STR_LEN_MAX]; + uint32_t tpn; + int ret; + + ret = udma_dfx_read_buf(str, p_buf); + if (ret) { + dev_info(udma_dev->dev, "the inputing is invalid\n"); + return ret; + } + + if (kstrtouint(str, 0, &tpn)) { + dev_err(udma_dev->dev, "convert str failed\n"); + return -EINVAL; + } + + ret = udma_dfx_query_context(udma_dev, tpn, &qp_context, + sizeof(qp_context), UDMA_CMD_QUERY_QPC); + if (ret) { + dev_err(udma_dev->dev, + "query qp context failed, ret = %d\n", ret); + return ret; + } + + udma_dfx_qpc_print(udma_dev, tpn, &qp_context); + return 0; +} + +static void udma_dfx_jfrc_print(struct udma_dev *udma_dev, uint32_t jfrn, + struct udma_jfr_context *jfr_context) +{ + uint32_t *jfrc = (uint32_t *)jfr_context; + uint32_t i; + + dev_info(udma_dev->dev, + "************ JFR/SRQ(0x%8x) CONTEXT INFO *************\n", + jfrn); + for (i = 0; i < (sizeof(*jfr_context) / sizeof(uint32_t)); i++) { + pr_info("SRQC(byte%4lu): %08x\n", (i + 1) * sizeof(uint32_t), + *jfrc); + jfrc++; + } + dev_info(udma_dev->dev, + "*********************************************************\n"); +} + +static int udma_dfx_jfr_store(const char *p_buf, struct udma_dfx_info *udma_dfx) +{ + struct udma_dev *udma_dev = (struct udma_dev *)udma_dfx->priv; + struct udma_jfr_context jfr_context; + char str[UDMA_DFX_STR_LEN_MAX]; + uint32_t jfrn; + int ret; + + ret = udma_dfx_read_buf(str, p_buf); + if (ret) { + dev_info(udma_dev->dev, "the inputing is invalid\n"); + return ret; + } + + if (kstrtouint(str, 0, &jfrn)) { + dev_err(udma_dev->dev, "convert str failed\n"); + return -EINVAL; + } + + ret = udma_dfx_query_context(udma_dev, jfrn, &jfr_context, + sizeof(jfr_context), UDMA_CMD_QUERY_SRQC); + if (ret) { + dev_err(udma_dev->dev, + "query jfr context failed, ret = %d\n", ret); + return ret; + } + + udma_dfx_jfrc_print(udma_dev, jfrn, &jfr_context); + return 0; +} + +static void udma_dfx_jfcc_print(struct udma_dev *udma_dev, uint32_t jfcn, + struct udma_jfc_context *jfc_context) +{ + int *jfcc = (int *)jfc_context; + uint32_t i; + + dev_info(udma_dev->dev, + "************ JFC/CQC(0x%8x) CONTEXT INFO *************\n", + jfcn); + for (i = 0; i < (sizeof(*jfc_context) / sizeof(int)); i++) { + pr_info("CQC(byte%4lu): %08x\n", (i + 1) * sizeof(int), *jfcc); + jfcc++; + } + dev_info(udma_dev->dev, + "*********************************************************\n"); +} + +static int udma_dfx_jfc_store(const char *p_buf, struct udma_dfx_info *udma_dfx) +{ + struct udma_dev *udma_dev = (struct udma_dev *)udma_dfx->priv; + struct udma_jfc_context jfc_context = {}; + char str[UDMA_DFX_STR_LEN_MAX] = {}; + uint32_t jfcn; + int ret; + + ret = udma_dfx_read_buf(str, p_buf); + if (ret) { + dev_info(udma_dev->dev, "the inputing is invalid\n"); + return ret; + } + + if (kstrtouint(str, 0, &jfcn)) { + dev_err(udma_dev->dev, "convert str failed\n"); + return -EINVAL; + } + + ret = udma_dfx_query_context(udma_dev, jfcn, &jfc_context, + sizeof(jfc_context), UDMA_CMD_QUERY_CQC); + if (ret) { + dev_info(udma_dev->dev, + "query jfc context fail, ret = %d, jfcn = %u\n", + ret, jfcn); + return ret; + } + + udma_dfx_jfcc_print(udma_dev, jfcn, &jfc_context); + return 0; +} + +int udma_find_dfx_dev(struct udma_dev *udma_dev, int *num) +{ + int i; + + for (i = 0; i < MAX_UDMA_DEV; i++) { + if (g_udma_dfx_list[i].dev == udma_dev) { + *num = i; + return 0; + } + } + + dev_err(udma_dev->dev, "failed to find dfx device!\n"); + return -EINVAL; +} + +static int udma_query_res_tp(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_tp_val *tp = (struct ubcore_res_tp_val *)val->addr; + struct udma_qp_context qp_context; + int ret; + + if (val->len < sizeof(struct ubcore_res_tp_val)) { + dev_err(udma_dev->dev, + "Failed to check len, type: %u, val->len: %u.\n", + (uint32_t)key->type, val->len); + val->len = sizeof(struct ubcore_res_tp_val); + return -EINVAL; + } + + ret = udma_dfx_query_context(udma_dev, key->key, &qp_context, + sizeof(qp_context), UDMA_CMD_QUERY_QPC); + if (ret) { + dev_err(udma_dev->dev, + "query qp context failed, ret = %d\n", ret); + return ret; + } + + tp->tpn = key->key; + tp->psn = udma_reg_read(&qp_context, QPC_SQ_CUR_PSN); + tp->pri = udma_reg_read(&qp_context, QPC_SL); + tp->oor = udma_reg_read(&qp_context.ext, QPCEX_OOR_EN); + tp->state = udma_reg_read(&qp_context, QPC_QP_ST); + tp->data_udp_start = udma_reg_read(&qp_context.ext, QPCEX_DATA_UDP_SRCPORT_L) | + udma_reg_read(&qp_context.ext, QPCEX_DATA_UDP_SRCPORT_H) << + QPCEX_DATA_UDP_SRCPORT_H_SHIFT; + tp->ack_udp_start = udma_reg_read(&qp_context.ext, QPCEX_ACK_UDP_SRCPORT); + tp->udp_range = udma_reg_read(&qp_context.ext, QPCEX_UDP_SRCPORT_RANGE); + tp->spray_en = udma_reg_read(&qp_context.ext, QPCEX_AR_EN); + + val->len = sizeof(struct ubcore_res_tp_val); + + return 0; +} + +static int udma_query_res_jfs(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_jfs_val *jfs = (struct ubcore_res_jfs_val *)val->addr; + struct jfs_list *jfs_now; + int ret; + int i; + + if (val->len < sizeof(struct ubcore_res_jfs_val)) { + dev_err(udma_dev->dev, + "Failed to check len, type: %u, val->len: %u.\n", + (uint32_t)key->type, val->len); + val->len = sizeof(struct ubcore_res_jfs_val); + return -EINVAL; + } + + ret = udma_find_dfx_dev(udma_dev, &i); + if (ret) + return ret; + + list_for_each_entry(jfs_now, + &g_udma_dfx_list[i].dfx->jfs_list->node, node) { + if (jfs_now->jfs_id == key->key) { + jfs->jfs_id = jfs_now->jfs_id; + jfs->state = jfs_now->state; + jfs->depth = jfs_now->depth; + jfs->pri = jfs_now->pri; + jfs->jfc_id = jfs_now->jfc_id; + val->len = sizeof(struct ubcore_res_jfs_val); + return 0; + } + } + + dev_err(udma_dev->dev, "failed to find jfs!\n"); + return -EINVAL; +} + +static int udma_query_res_jfr(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_jfr_val *jfr = (struct ubcore_res_jfr_val *)val->addr; + struct udma_jfr_context jfr_context; + struct jfr_list *jfr_now; + int ret; + int i; + + if (val->len < sizeof(struct ubcore_res_jfr_val)) { + dev_err(udma_dev->dev, + "Failed to check len, type: %u, val->len: %u.\n", + (uint32_t)key->type, val->len); + val->len = sizeof(struct ubcore_res_jfr_val); + return -EINVAL; + } + + ret = udma_dfx_query_context(udma_dev, key->key, &jfr_context, + sizeof(jfr_context), UDMA_CMD_QUERY_SRQC); + if (ret) { + dev_err(udma_dev->dev, + "query jfr context failed, ret = %d\n", ret); + return ret; + } + + ret = udma_find_dfx_dev(udma_dev, &i); + if (ret) + return ret; + + list_for_each_entry(jfr_now, + &g_udma_dfx_list[i].dfx->jfr_list->node, node) { + if (jfr_now->jfr_id == key->key) { + jfr->jfr_id = key->key; + jfr->state = udma_reg_read(&jfr_context, SRQC_SRQ_ST); + jfr->depth = 1 << udma_reg_read(&jfr_context, SRQC_SHIFT); + jfr->jfc_id = jfr_now->jfc_id; + val->len = sizeof(struct ubcore_res_jfr_val); + return 0; + } + } + + dev_err(udma_dev->dev, "failed to find jfr!\n"); + return -EINVAL; +} + +static int udma_query_res_jetty(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_jetty_val *jetty = (struct ubcore_res_jetty_val *)val->addr; + struct jetty_list *jetty_now; + int ret; + int i; + + if (val->len < sizeof(struct ubcore_res_jetty_val)) { + dev_err(udma_dev->dev, + "Failed to check len, type: %u, val->len: %u.\n", + (uint32_t)key->type, val->len); + val->len = sizeof(struct ubcore_res_jetty_val); + return -EINVAL; + } + + ret = udma_find_dfx_dev(udma_dev, &i); + if (ret) + return ret; + + list_for_each_entry(jetty_now, + &g_udma_dfx_list[i].dfx->jetty_list->node, node) { + if (jetty_now->jetty_id == key->key) { + jetty->jetty_id = jetty_now->jetty_id; + jetty->state = jetty_now->state; + jetty->jfs_depth = jetty_now->jfs_depth; + jetty->jfr_depth = jetty_now->jfr_depth; + jetty->pri = jetty_now->pri; + jetty->jfr_id = jetty_now->jfr_id; + jetty->send_jfc_id = jetty_now->jfc_s_id; + jetty->recv_jfc_id = jetty_now->jfc_r_id; + val->len = sizeof(struct ubcore_res_jetty_val); + return 0; + } + } + + dev_err(udma_dev->dev, "failed to find jetty!\n"); + return -EINVAL; +} + +static int udma_query_res_jfc(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_jfc_val *jfc = (struct ubcore_res_jfc_val *)val->addr; + struct udma_jfc_context jfc_context; + int ret; + + if (val->len < sizeof(struct ubcore_res_jfc_val)) { + dev_err(udma_dev->dev, + "Failed to check len, type: %u, val->len: %u.\n", + (uint32_t)key->type, val->len); + val->len = sizeof(struct ubcore_res_jfc_val); + return -EINVAL; + } + + ret = udma_dfx_query_context(udma_dev, key->key, &jfc_context, + sizeof(jfc_context), UDMA_CMD_QUERY_CQC); + if (ret) { + dev_err(udma_dev->dev, + "query jfc context failed, ret = %d\n", ret); + return ret; + } + + jfc->jfc_id = udma_reg_read(&jfc_context, CQC_CQN); + jfc->state = udma_reg_read(&jfc_context, CQC_CQ_ST); + jfc->depth = 1 << udma_reg_read(&jfc_context, CQC_SHIFT); + + val->len = sizeof(struct ubcore_res_jfc_val); + + return 0; +} + +static int udma_query_res_seg(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_seg_val *seg = (struct ubcore_res_seg_val *)val->addr; + struct udma_mpt_entry mpt_entry; + uint32_t mpt_index; + int ret; + + if (val->len < sizeof(struct ubcore_res_seg_val)) { + dev_err(udma_dev->dev, + "Failed to check len, type: %u, val->len: %u.\n", + (uint32_t)key->type, val->len); + val->len = sizeof(struct ubcore_res_seg_val); + return -EINVAL; + } + + mpt_index = key_to_hw_index(key->key) & (udma_dev->caps.num_mtpts - 1); + ret = udma_dfx_query_context(udma_dev, mpt_index, &mpt_entry, + sizeof(mpt_entry), UDMA_CMD_QUERY_MPT); + if (ret) { + dev_err(udma_dev->dev, + "query seg context failed, ret = %d\n", ret); + return ret; + } + + seg->ubva.eid = udma_dev->ub_dev.attr.eid; + seg->ubva.uasid = udma_reg_read(&mpt_entry, MPT_PD); + seg->ubva.va = udma_reg_read(&mpt_entry, MPT_VA_L) | + udma_reg_read(&mpt_entry, MPT_VA_H) << + MPT_VA_H_SHIFT; + seg->len = udma_reg_read(&mpt_entry, MPT_LEN_L) | + udma_reg_read(&mpt_entry, MPT_LEN_H) << + MPT_LEN_H_SHIFT; + seg->key_id = udma_reg_read(&mpt_entry, MPT_LKEY); + + val->len = sizeof(struct ubcore_res_seg_val); + + return 0; +} + +static int udma_query_res_dev_tp(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val, int i) +{ + struct ubcore_res_dev_val *dev = (struct ubcore_res_dev_val *)val->addr; + uint32_t *tp_list_ptr = dev->tp_list; + struct tpn_list *tpn_now; + + dev->tp_cnt = 0; + if (!g_udma_dfx_list[i].dfx) { + dev_err(udma_dev->dev, "query res_dev_tp failed!\n"); + return -EINVAL; + } + + list_for_each_entry(tpn_now, + &g_udma_dfx_list[i].dfx->tpn_list->node, node) { + *tp_list_ptr = tpn_now->tpn; + tp_list_ptr++; + dev->tp_cnt++; + if (dev->tp_cnt == MAX_TP_CNT) + break; + } + + return 0; +} + +static int udma_query_res_dev_jfs(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val, int i) +{ + struct ubcore_res_dev_val *dev = (struct ubcore_res_dev_val *)val->addr; + uint32_t *jfs_list_ptr = dev->jfs_list; + struct jfs_list *jfs_now; + + dev->jfs_cnt = 0; + if (!g_udma_dfx_list[i].dfx) { + dev_err(udma_dev->dev, "query res_dev_jfs failed!\n"); + return -EINVAL; + } + + list_for_each_entry(jfs_now, + &g_udma_dfx_list[i].dfx->jfs_list->node, node) { + *jfs_list_ptr = jfs_now->jfs_id; + jfs_list_ptr++; + dev->jfs_cnt++; + if (dev->jfs_cnt == MAX_JFS_CNT) + break; + } + return 0; +} + +static int udma_query_res_dev_jfr(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val, int i) +{ + struct ubcore_res_dev_val *dev = (struct ubcore_res_dev_val *)val->addr; + uint32_t *jfr_list_ptr = dev->jfr_list; + struct jfr_list *jfr_now; + + dev->jfr_cnt = 0; + if (!g_udma_dfx_list[i].dfx) { + dev_err(udma_dev->dev, "query res_dev_jfr failed!\n"); + return -EINVAL; + } + + list_for_each_entry(jfr_now, + &g_udma_dfx_list[i].dfx->jfr_list->node, node) { + *jfr_list_ptr = jfr_now->jfr_id; + jfr_list_ptr++; + dev->jfr_cnt++; + if (dev->jfr_cnt == MAX_JFR_CNT) + break; + } + return 0; +} + +static int udma_query_res_dev_jetty(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val, int i) +{ + struct ubcore_res_dev_val *dev = (struct ubcore_res_dev_val *)val->addr; + uint32_t *jetty_list_ptr = dev->jetty_list; + struct jetty_list *jetty_now; + + dev->jetty_cnt = 0; + if (!g_udma_dfx_list[i].dfx) { + dev_err(udma_dev->dev, "query res_dev_jetty failed!\n"); + return -EINVAL; + } + + list_for_each_entry(jetty_now, + &g_udma_dfx_list[i].dfx->jetty_list->node, node) { + *jetty_list_ptr = jetty_now->jetty_id; + jetty_list_ptr++; + dev->jetty_cnt++; + if (dev->jetty_cnt == MAX_JETTY_CNT) + break; + } + return 0; +} + +static int udma_query_res_dev_jfc(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val, int i) +{ + struct ubcore_res_dev_val *dev = (struct ubcore_res_dev_val *)val->addr; + uint32_t *jfc_list_ptr = dev->jfc_list; + struct jfc_list *jfc_now; + + dev->jfc_cnt = 0; + if (!g_udma_dfx_list[i].dfx) { + dev_err(udma_dev->dev, "query res_dev_jfc failed!\n"); + return -EINVAL; + } + + list_for_each_entry(jfc_now, + &g_udma_dfx_list[i].dfx->jfc_list->node, node) { + *jfc_list_ptr = jfc_now->jfc_id; + jfc_list_ptr++; + dev->jfc_cnt++; + if (dev->jfc_cnt == MAX_JFC_CNT) + break; + } + return 0; +} + +static int udma_query_res_dev_seg(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val, int i) +{ + struct ubcore_res_dev_val *dev = (struct ubcore_res_dev_val *)val->addr; + struct ubcore_seg_info *seg_list_ptr = dev->seg_list; + struct seg_list *seg_now; + + dev->seg_cnt = 0; + if (!g_udma_dfx_list[i].dfx) { + dev_err(udma_dev->dev, "query res_dev_seg failed!\n"); + return -EINVAL; + } + + list_for_each_entry(seg_now, + &g_udma_dfx_list[i].dfx->seg_list->node, node) { + seg_list_ptr->ubva.eid = udma_dev->ub_dev.attr.eid; + seg_list_ptr->ubva.uasid = seg_now->pd; + seg_list_ptr->ubva.va = seg_now->iova; + seg_list_ptr->len = seg_now->len; + seg_list_ptr->key_id = seg_now->key_id; + seg_list_ptr++; + dev->seg_cnt++; + if (dev->seg_cnt == MAX_SEG_CNT) + break; + } + return 0; +} + +static int udma_query_res_dev(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_dev_val *dev = (struct ubcore_res_dev_val *)val->addr; + int ret; + int i; + + if (val->len < sizeof(struct ubcore_res_dev_val)) { + dev_err(udma_dev->dev, + "Failed to check len, type: %u, val->len: %u.\n", + (uint32_t)key->type, val->len); + val->len = sizeof(struct ubcore_res_dev_val); + return -EINVAL; + } + + ret = udma_find_dfx_dev(udma_dev, &i); + if (ret) + return ret; + + ret = udma_query_res_dev_tp(udma_dev, key, val, i); + if (ret) + return ret; + + ret = udma_query_res_dev_jfs(udma_dev, key, val, i); + if (ret) + return ret; + + ret = udma_query_res_dev_jfr(udma_dev, key, val, i); + if (ret) + return ret; + + ret = udma_query_res_dev_jetty(udma_dev, key, val, i); + if (ret) + return ret; + + ret = udma_query_res_dev_jfc(udma_dev, key, val, i); + if (ret) + return ret; + + ret = udma_query_res_dev_seg(udma_dev, key, val, i); + if (ret) + return ret; + + dev->tpg_cnt = 0; + dev->utp_cnt = 0; + dev->jetty_group_cnt = 0; + + return 0; +} + +static int udma_check_key_type(struct udma_dev *udma_dev, + struct ubcore_res_key *key) +{ + bool ret; + + ret = key->type < UBCORE_RES_KEY_UPI || + key->type > UBCORE_RES_KEY_URMA_DEV || + key->type == UBCORE_RES_KEY_UPI || + key->type == UBCORE_RES_KEY_TPG || + key->type == UBCORE_RES_KEY_UTP || + key->type == UBCORE_RES_KEY_JETTY_GROUP; + if (ret) { + dev_err(udma_dev->dev, + "Key type: %u invalid.\n", (uint32_t)key->type); + return -EINVAL; + } + + return 0; +} + +int udma_query_res(const struct ubcore_device *dev, + struct ubcore_res_key *key, struct ubcore_res_val *val) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + int ret; + + ret = udma_check_key_type(udma_dev, key); + if (ret) + return ret; + + switch (key->type) { + case UBCORE_RES_KEY_TP: + return udma_query_res_tp(udma_dev, key, val); + case UBCORE_RES_KEY_JFS: + return udma_query_res_jfs(udma_dev, key, val); + case UBCORE_RES_KEY_JFR: + return udma_query_res_jfr(udma_dev, key, val); + case UBCORE_RES_KEY_JETTY: + return udma_query_res_jetty(udma_dev, key, val); + case UBCORE_RES_KEY_JFC: + return udma_query_res_jfc(udma_dev, key, val); + case UBCORE_RES_KEY_SEG: + return udma_query_res_seg(udma_dev, key, val); + case UBCORE_RES_KEY_URMA_DEV: + return udma_query_res_dev(udma_dev, key, val); + default: + return -EINVAL; + } + + return 0; +} + +UDMA_DFX_FILE_ATTR_DEF(tp_context, NULL, udma_dfx_tp_store); +UDMA_DFX_FILE_ATTR_DEF(jfr_context, NULL, udma_dfx_jfr_store); +UDMA_DFX_FILE_ATTR_DEF(seg_context, NULL, udma_dfx_seg_store); +UDMA_DFX_FILE_ATTR_DEF(jfc_context, NULL, udma_dfx_jfc_store); + +static struct attribute *udma_dfx_attrs_list[] = { + HW_ATTRS_LIST_MEMBER(tp_context), + HW_ATTRS_LIST_MEMBER(jfr_context), + HW_ATTRS_LIST_MEMBER(seg_context), + HW_ATTRS_LIST_MEMBER(jfc_context), + NULL +}; + +static ssize_t udma_dfx_show(struct kobject *kobj, struct attribute *attr, + char *buf) +{ + struct udma_dfx_sys_attr *p_udma_sys_attr = + container_of(attr, struct udma_dfx_sys_attr, attr); + struct udma_dfx_info *udma_dfx = container_of(kobj, + struct udma_dfx_info, + kobj); + int ret; + + memset(buf, 0, PAGE_SIZE); + if (p_udma_sys_attr->pub_show) { + ret = p_udma_sys_attr->pub_show(udma_dfx); + if (ret) + return ret; + else + return strlen(buf); + } + + return -EPERM; +} + +static ssize_t udma_dfx_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count) +{ + struct udma_dfx_sys_attr *p_udma_sys_attr = + container_of(attr, struct udma_dfx_sys_attr, attr); + struct udma_dfx_info *udma_dfx = container_of(kobj, + struct udma_dfx_info, + kobj); + int ret; + + if (p_udma_sys_attr->pub_store) { + ret = p_udma_sys_attr->pub_store((char *)buf, udma_dfx); + if (ret) + return ret; + else + return count; + } + + return -EPERM; +} + +static const struct sysfs_ops udma_dfx_file_ops = { + .show = udma_dfx_show, + .store = udma_dfx_store, +}; + +static struct kobj_type udma_dfx_kobj_ktype = { + .release = NULL, + .sysfs_ops = &udma_dfx_file_ops, + .default_attrs = udma_dfx_attrs_list, +}; + +static int udma_dfx_add_sysfs(struct udma_dfx_info *udma_dfx) +{ + struct device *dev = udma_dfx->drv_dev; + int ret; + + ret = kobject_init_and_add(&udma_dfx->kobj, + &udma_dfx_kobj_ktype, + &dev->kobj, + "%s", udma_dfx->dev.dev_name); + if (ret) + dev_err(drv_device, "kobject_init_and_add failed!\r\n"); + + return ret; +} + +static void udma_dfx_del_sysfs(struct udma_dfx_info *udma_dfx) +{ + kobject_del(&udma_dfx->kobj); +} + +struct udma_dfx_ops udma_dfx_ops = { + .add_sysfs = udma_dfx_add_sysfs, + .del_sysfs = udma_dfx_del_sysfs, +}; + +static void list_lock_init(struct udma_dfx_info *dfx) +{ + spin_lock_init(&dfx->tpn_list->node_lock); + INIT_LIST_HEAD(&dfx->tpn_list->node); + spin_lock_init(&dfx->jfs_list->node_lock); + INIT_LIST_HEAD(&dfx->jfs_list->node); + spin_lock_init(&dfx->jfr_list->node_lock); + INIT_LIST_HEAD(&dfx->jfr_list->node); + spin_lock_init(&dfx->jetty_list->node_lock); + INIT_LIST_HEAD(&dfx->jetty_list->node); + spin_lock_init(&dfx->jfc_list->node_lock); + INIT_LIST_HEAD(&dfx->jfc_list->node); + spin_lock_init(&dfx->seg_list->node_lock); + INIT_LIST_HEAD(&dfx->seg_list->node); +} + +static int udma_dfx_list_init(int num) +{ + struct udma_dfx_info *dfx; + int ret = -ENOMEM; + + dfx = g_udma_dfx_list[num].dfx; + + dfx->tpn_list = kzalloc(sizeof(struct tpn_list), GFP_KERNEL); + if (!dfx->tpn_list) + return ret; + + dfx->jfs_list = kzalloc(sizeof(struct jfs_list), GFP_KERNEL); + if (!dfx->jfs_list) + goto tpn_list_alloc_failed; + + dfx->jfr_list = kzalloc(sizeof(struct jfr_list), GFP_KERNEL); + if (!dfx->jfr_list) + goto jfs_id_list_alloc_failed; + + dfx->jetty_list = kzalloc(sizeof(struct jetty_list), GFP_KERNEL); + if (!dfx->jetty_list) + goto jfr_id_list_alloc_failed; + + dfx->jfc_list = kzalloc(sizeof(struct jfc_list), GFP_KERNEL); + if (!dfx->jfc_list) + goto jetty_id_list_alloc_failed; + + dfx->seg_list = kzalloc(sizeof(struct seg_list), GFP_KERNEL); + if (!dfx->seg_list) + goto jfc_id_list_alloc_failed; + + list_lock_init(dfx); + + return 0; + +jfc_id_list_alloc_failed: + kfree(dfx->jfc_list); +jetty_id_list_alloc_failed: + kfree(dfx->jetty_list); +jfr_id_list_alloc_failed: + kfree(dfx->jfr_list); +jfs_id_list_alloc_failed: + kfree(dfx->jfs_list); +tpn_list_alloc_failed: + kfree(dfx->tpn_list); + dev_err(drv_device, "dfx alloc list failed\n"); + + return ret; +} + +#define DFX_LIST_FREE(name) \ +do { \ + lock = &dfx->name##_list->node_lock; \ + spin_lock_irqsave(lock, flags); \ + list_for_each_entry_safe(name##_id, name##_tmp, \ + &dfx->name##_list->node, node) { \ + list_del(&name##_id->node); \ + kfree(name##_id); \ + } \ + spin_unlock_irqrestore(lock, flags); \ +} while (0) + +static void udma_dfx_list_free(int num) +{ + struct jetty_list *jetty_id, *jetty_tmp; + struct jfs_list *jfs_id, *jfs_tmp; + struct jfr_list *jfr_id, *jfr_tmp; + struct jfc_list *jfc_id, *jfc_tmp; + struct seg_list *seg_id, *seg_tmp; + struct tpn_list *tpn_id, *tpn_tmp; + struct udma_dfx_info *dfx; + unsigned long flags; + spinlock_t *lock; + + dfx = g_udma_dfx_list[num].dfx; + DFX_LIST_FREE(jetty); + DFX_LIST_FREE(tpn); + DFX_LIST_FREE(jfr); + DFX_LIST_FREE(jfs); + DFX_LIST_FREE(seg); + DFX_LIST_FREE(jfc); +} + +static int udma_dfx_add_udma_device(struct udma_dev *udma_dev) +{ + int ret; + int i; + + if (udma_dev_count == MAX_UDMA_DEV) { + dev_err(drv_device, + "udma dfx add device failed, g_udma_dfx_list is full\n."); + ret = -EINVAL; + goto g_udma_dfx_list_full; + } + for (i = 0; i < MAX_UDMA_DEV; i++) + if (!g_udma_dfx_list[i].dfx) + break; + + g_udma_dfx_list[i].dev = udma_dev; + g_udma_dfx_list[i].dfx = kzalloc(sizeof(struct udma_dfx_info), + GFP_KERNEL); + if (!g_udma_dfx_list[i].dfx) { + ret = -ENOMEM; + goto dfx_info_alloc_failed; + } + g_udma_dfx_list[i].dfx->priv = (void *)udma_dev; + g_udma_dfx_list[i].dfx->ops = &udma_dfx_ops; + g_udma_dfx_list[i].dfx->drv_dev = drv_device; + strlcpy(g_udma_dfx_list[i].dfx->dev.dev_name, udma_dev->dev_name, + UBCORE_MAX_DEV_NAME); + ret = udma_dfx_list_init(i); + if (ret) { + dev_err(drv_device, "dfx add dev list failed\n"); + goto dfx_info_alloc_failed; + } + + ret = g_udma_dfx_list[i].dfx->ops->add_sysfs(g_udma_dfx_list[i].dfx); + if (ret) { + dev_err(drv_device, "dfx add sysfs failed\n"); + goto add_sysfs_failed; + } + + dev_info(drv_device, "add udma device (%s) in udma dfx\n", + g_udma_dfx_list[i].dfx->dev.dev_name); + + udma_dev_count++; + + return 0; + +add_sysfs_failed: + udma_dfx_list_free(i); + kfree(g_udma_dfx_list[i].dfx); + g_udma_dfx_list[i].dfx = NULL; +dfx_info_alloc_failed: + g_udma_dfx_list[i].dev = NULL; +g_udma_dfx_list_full: + return ret; +} + +static int udma_dfx_chrdev_create(struct udma_dev *udma_dev) +{ + int ret; + + major = register_chrdev(0, DFX_DEVICE_NAME, &chr_ops); + if (major < 0) { + dev_err(udma_dev->dev, + "udma dfx register the character device failed\n "); + ret = major; + goto device_register_failed; + } + + drv_class = class_create(THIS_MODULE, DFX_DEVICE_NAME); + if (IS_ERR(drv_class)) { + dev_err(udma_dev->dev, "udma dfx class create failed\n"); + ret = (int)PTR_ERR(drv_class); + goto class_create_failed; + } + + drv_device = device_create(drv_class, NULL, MKDEV(major, 0), + NULL, DFX_DEVICE_NAME); + if (IS_ERR(drv_device)) { + dev_err(udma_dev->dev, "udma dfx create device failed\n"); + ret = (int)PTR_ERR(drv_device); + goto device_create_failed; + } + + return 0; + +device_create_failed: + drv_device = NULL; + class_destroy(drv_class); +class_create_failed: + drv_class = NULL; + unregister_chrdev(major, DFX_DEVICE_NAME); +device_register_failed: + major = -1; + return ret; +} + +static void udma_dfx_chrdev_destroy(void) +{ + device_destroy(drv_class, MKDEV(major, 0)); + drv_device = NULL; + class_destroy(drv_class); + drv_class = NULL; + unregister_chrdev(major, DFX_DEVICE_NAME); + major = -1; +} + +int udma_dfx_init(struct udma_dev *udma_dev) +{ + int ret; + + if (!udma_dev_count) { + ret = udma_dfx_chrdev_create(udma_dev); + if (ret) { + dev_err(drv_device, + "udma dfx create chr device failed\n"); + goto chrdev_create_failed; + } + dev_info(drv_device, "udma dfx create chr device success\n"); + } + + ret = udma_dfx_add_udma_device(udma_dev); + if (ret) { + dev_err(drv_device, "udma dfx add udma device failed\n"); + goto add_device_failed; + } + + return 0; + +add_device_failed: + if (!udma_dev_count) { + dev_info(drv_device, "udma dfx remove chr device\n"); + udma_dfx_chrdev_destroy(); + } +chrdev_create_failed: + return ret; +} + +static void udma_dfx_remove_udma_device(struct udma_dev *udma_dev) +{ + int i; + + for (i = 0; i < MAX_UDMA_DEV; i++) { + if (g_udma_dfx_list[i].dev) { + if (g_udma_dfx_list[i].dev == udma_dev) { + dev_info(drv_device, + "rmv udma device (%s) from udma dfx\n", + g_udma_dfx_list[i].dfx->dev.dev_name); + g_udma_dfx_list[i].dfx->ops->del_sysfs(g_udma_dfx_list[i].dfx); + udma_dfx_list_free(i); + + kfree(g_udma_dfx_list[i].dfx); + g_udma_dfx_list[i].dfx = NULL; + g_udma_dfx_list[i].dev = NULL; + udma_dev_count--; + break; + } + } + } +} + +void udma_dfx_uninit(struct udma_dev *udma_dev) +{ + if (!udma_dev_count) { + dev_err(udma_dev->dev, "no udma dfx device\n"); + return; + } + + udma_dfx_remove_udma_device(udma_dev); + if (!udma_dev_count) { + dev_info(drv_device, "udma dfx remove chr device\n"); + udma_dfx_chrdev_destroy(); + } +} diff --git a/drivers/ub/hw/hns3/hns3_udma_dfx.h b/drivers/ub/hw/hns3/hns3_udma_dfx.h new file mode 100644 index 000000000000..faacd0a536f9 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_dfx.h @@ -0,0 +1,139 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_DFX_H +#define _UDMA_DFX_H + +#include +#include "hns3_udma_device.h" + +#define DFX_DEVICE_NAME "udma_dfx" +#define MAX_UDMA_DEV 16 +#define MAX_TP_CNT 256 +#define MAX_JFS_CNT 256 +#define MAX_JFR_CNT 256 +#define MAX_JETTY_CNT 256 +#define MAX_JFC_CNT 256 +#define MAX_SEG_CNT 256 +#define UDMA_DFX_FILE_ATTR_DEF(file_name, func_show, func_store) \ +static struct udma_dfx_sys_attr g_sysfs_udma_##file_name##_attr = {\ + {\ + .name = #file_name,\ + .mode = 0644,\ + },\ + .pub_show = (func_show),\ + .pub_store = (func_store),\ +} + +#define HW_ATTRS_LIST_MEMBER(file_name) (&g_sysfs_udma_##file_name##_attr.attr) +#define MAX_CHAR_NUM_DEV_NAME 12 +#define UDMA_DFX_STR_LEN_MAX 20 + +struct udma_dfx_info; + +struct udma_dfx_sys_attr { + struct attribute attr; + int (*pub_show)(struct udma_dfx_info *udma_dfx); + int (*pub_store)(const char *buf, struct udma_dfx_info *udma_dfx); +}; + +struct udma_dfx_dev_info { + char dev_name[MAX_CHAR_NUM_DEV_NAME]; +}; + +struct udma_dfx_ops { + int (*add_sysfs)(struct udma_dfx_info *info); + void (*del_sysfs)(struct udma_dfx_info *info); +}; + +struct tpn_list { + uint32_t tpn; + struct list_head node; + spinlock_t node_lock; +}; + +struct jfs_list { + uint32_t jfs_id; + uint8_t state; + uint16_t depth; + uint8_t pri; + uint32_t jfc_id; + struct list_head node; + spinlock_t node_lock; +}; + +struct jfr_list { + uint32_t jfr_id; + uint32_t jfc_id; + struct list_head node; + spinlock_t node_lock; +}; + +struct jetty_list { + uint32_t jetty_id; + uint8_t state; + uint32_t jfs_depth; + uint32_t jfr_depth; + uint8_t pri; + uint32_t jfc_s_id; + uint32_t jfc_r_id; + uint32_t jfr_id; + struct list_head node; + spinlock_t node_lock; +}; + +struct jfc_list { + uint32_t jfc_id; + struct list_head node; + spinlock_t node_lock; +}; + +struct seg_list { + uint32_t pd; + uint64_t iova; + uint32_t len; + uint32_t key_id; + struct list_head node; + spinlock_t node_lock; +}; + +struct udma_dfx_info { + struct udma_dfx_dev_info dev; + struct udma_dfx_ops *ops; + struct device *drv_dev; + struct kobject kobj; + struct tpn_list *tpn_list; + struct jfs_list *jfs_list; + struct jfr_list *jfr_list; + struct jetty_list *jetty_list; + struct jfc_list *jfc_list; + struct seg_list *seg_list; + void *priv; +}; + +struct udma_dfx_dev { + struct udma_dfx_info *dfx; + struct udma_dev *dev; +}; + +extern struct udma_dfx_dev g_udma_dfx_list[MAX_UDMA_DEV]; + +int udma_dfx_init(struct udma_dev *udma_dev); +void udma_dfx_uninit(struct udma_dev *udma_dev); +int udma_find_dfx_dev(struct udma_dev *udma_dev, int *num); +int udma_query_res(const struct ubcore_device *dev, struct ubcore_res_key *key, + struct ubcore_res_val *val); + +#endif /* _UDMA_DFX_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_eq.c b/drivers/ub/hw/hns3/hns3_udma_eq.c index 9fcd321e9ccb..0710f130a463 100644 --- a/drivers/ub/hw/hns3/hns3_udma_eq.c +++ b/drivers/ub/hw/hns3/hns3_udma_eq.c @@ -339,6 +339,7 @@ static int udma_aeq_int(struct udma_dev *udma_dev, struct udma_eq *eq) ++eq->cons_index; + udma_dev->dfx_cnt[UDMA_DFX_AEQE]++; BUILD_BUG_ON(sizeof(struct udma_aeqe) > TRACE_AEQE_LEN_MAX); udma_init_irq_work(udma_dev, eq, aeqe, queue_num); @@ -381,6 +382,7 @@ static int udma_ceq_int(struct udma_dev *udma_dev, udma_jfc_completion(udma_dev, cqn); ++eq->cons_index; + udma_dev->dfx_cnt[UDMA_DFX_CEQE]++; ceqe_found = 1; ceqe = next_ceqe_sw_v2(eq); diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.c b/drivers/ub/hw/hns3/hns3_udma_hw.c index e5b588c5b6a3..b050ce2fe003 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hw.c +++ b/drivers/ub/hw/hns3/hns3_udma_hw.c @@ -22,8 +22,11 @@ #include "hns3_udma_hem.h" #include "hns3_udma_eq.h" #include "hns3_udma_qp.h" +#include "hns3_udma_dfx.h" #include "hns3_udma_sysfs.h" +bool dfx_switch; + static const struct pci_device_id udma_hw_pci_tbl[] = { { PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_UDMA_OVER_UBL), HNAE3_DEV_SUPPORT_UDMA_OVER_UBL_DCB_BITS }, @@ -1867,6 +1870,15 @@ static int __udma_init_instance(struct hnae3_handle *handle) } handle->priv = udma_dev; + if (dfx_switch) { + ret = udma_dfx_init(udma_dev); + if (ret) { + dev_err(udma_dev->dev, "UDMA dfx init failed(%d)!\n", + ret); + goto error_failed_dfx_init; + } + } + ret = udma_register_cc_sysfs(udma_dev); if (ret) { dev_err(udma_dev->dev, @@ -1876,6 +1888,9 @@ static int __udma_init_instance(struct hnae3_handle *handle) return 0; error_failed_cc_sysfs: + if (dfx_switch) + udma_dfx_uninit(udma_dev); +error_failed_dfx_init: udma_hnae_client_exit(udma_dev); error_failed_get_cfg: kfree(udma_dev->priv); @@ -1894,6 +1909,10 @@ static void __udma_uninit_instance(struct hnae3_handle *handle, return; udma_unregister_cc_sysfs(udma_dev); + + if (dfx_switch) + udma_dfx_uninit(handle->priv); + handle->priv = NULL; udma_hnae_client_exit(udma_dev); @@ -2117,3 +2136,7 @@ module_exit(udma_exit); MODULE_LICENSE("Dual BSD/GPL"); MODULE_DESCRIPTION("UBUS UDMA Driver"); + +module_param(dfx_switch, bool, 0444); +MODULE_PARM_DESC(dfx_switch, + "Set whether to enable the udma_dfx, default: 0(0:off, 1:on)"); diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.h b/drivers/ub/hw/hns3/hns3_udma_hw.h index 6c6bdc372a30..cb970fb99fab 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hw.h +++ b/drivers/ub/hw/hns3/hns3_udma_hw.h @@ -18,6 +18,8 @@ #include "urma/ubcore_types.h" +extern bool dfx_switch; + struct udma_query_version { uint16_t udma_vendor_id; uint16_t udma_hw_version; diff --git a/drivers/ub/hw/hns3/hns3_udma_jetty.c b/drivers/ub/hw/hns3/hns3_udma_jetty.c index 2684a9a554c0..31d40200c32c 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jetty.c +++ b/drivers/ub/hw/hns3/hns3_udma_jetty.c @@ -19,6 +19,7 @@ #include "hns3_udma_jfc.h" #include "hns3_udma_jfr.h" #include "hns3_udma_db.h" +#include "hns3_udma_dfx.h" #include "hns3_udma_jetty.h" static void init_jetty_cfg(struct udma_jetty *jetty, @@ -299,6 +300,85 @@ static int alloc_jetty_id(struct udma_dev *udma_dev, struct udma_jetty *jetty) return ret; } +static void store_jetty_id(struct udma_dev *udma_dev, struct udma_jetty *jetty) +{ + struct jetty_list *jetty_new; + struct jetty_list *jetty_now; + unsigned long flags; + spinlock_t *lock; + int ret; + int i; + + ret = udma_find_dfx_dev(udma_dev, &i); + if (ret) + return; + + jetty_new = kzalloc(sizeof(struct jetty_list), GFP_KERNEL); + if (jetty_new == NULL) + return; + + lock = &g_udma_dfx_list[i].dfx->jetty_list->node_lock; + spin_lock_irqsave(lock, flags); + list_for_each_entry(jetty_now, + &g_udma_dfx_list[i].dfx->jetty_list->node, node) { + if (jetty_now->jetty_id == jetty->jetty_id) { + jetty_now->jfs_depth = + jetty->ubcore_jetty.jetty_cfg.jfs_depth; + jetty_now->jfr_depth = + jetty->ubcore_jetty.jetty_cfg.jfr_depth; + jetty_now->pri = + jetty->ubcore_jetty.jetty_cfg.priority; + jetty_now->jfr_id = jetty->udma_jfr->jfrn; + jetty_now->jfc_s_id = jetty->ubcore_jetty.jetty_cfg.send_jfc->id; + jetty_now->jfc_r_id = jetty->ubcore_jetty.jetty_cfg.recv_jfc->id; + goto found; + } + } + + jetty_new->jetty_id = jetty->jetty_id; + jetty_new->jfs_depth = jetty->ubcore_jetty.jetty_cfg.jfs_depth; + jetty_new->jfr_depth = jetty->ubcore_jetty.jetty_cfg.jfr_depth; + jetty_new->pri = jetty->ubcore_jetty.jetty_cfg.priority; + jetty_new->jfr_id = jetty->udma_jfr->jfrn; + jetty_new->jfc_s_id = jetty->ubcore_jetty.jetty_cfg.send_jfc->id; + jetty_new->jfc_r_id = jetty->ubcore_jetty.jetty_cfg.recv_jfc->id; + list_add(&jetty_new->node, &g_udma_dfx_list[i].dfx->jetty_list->node); + spin_unlock_irqrestore(lock, flags); + + return; + +found: + spin_unlock_irqrestore(lock, flags); + kfree(jetty_new); +} + +static void delete_jetty_id(struct udma_dev *udma_dev, + struct udma_jetty *jetty) +{ + struct jetty_list *jetty_now, *jetty_tmp; + unsigned long flags; + spinlock_t *lock; + int ret; + int i; + + ret = udma_find_dfx_dev(udma_dev, &i); + if (ret) + return; + + lock = &g_udma_dfx_list[i].dfx->jetty_list->node_lock; + spin_lock_irqsave(lock, flags); + list_for_each_entry_safe(jetty_now, jetty_tmp, + &g_udma_dfx_list[i].dfx->jetty_list->node, + node) { + if (jetty_now->jetty_id == jetty->jetty_id) { + list_del(&jetty_now->node); + kfree(jetty_now); + spin_unlock_irqrestore(lock, flags); + return; + } + } + spin_unlock_irqrestore(lock, flags); +} static void free_jetty_id(struct udma_dev *udma_dev, struct udma_jetty *jetty) { @@ -338,6 +418,9 @@ struct ubcore_jetty *udma_create_jetty(struct ubcore_device *dev, mutex_init(&jetty->tp_mutex); + if (dfx_switch) + store_jetty_id(udma_dev, jetty); + return &jetty->ubcore_jetty; err_alloc_jetty_buf: @@ -380,6 +463,10 @@ int udma_destroy_jetty(struct ubcore_jetty *jetty) ret = free_jetty_buf(udma_dev, udma_jetty); clean_jetty_x_qpn_bitmap(&udma_jetty->qpn_map); + + if (dfx_switch) + delete_jetty_id(udma_dev, udma_jetty); + free_jetty_id(udma_dev, udma_jetty); kfree(udma_jetty); diff --git a/drivers/ub/hw/hns3/hns3_udma_jfc.c b/drivers/ub/hw/hns3/hns3_udma_jfc.c index f4263e82e46b..4abb9ad2e84f 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jfc.c +++ b/drivers/ub/hw/hns3/hns3_udma_jfc.c @@ -17,6 +17,7 @@ #include "hns3_udma_hem.h" #include "hns3_udma_cmd.h" #include "hns3_udma_db.h" +#include "hns3_udma_dfx.h" #include "hns3_udma_jfc.h" static int udma_hw_create_cq(struct udma_dev *dev, @@ -411,6 +412,69 @@ static int alloc_jfc_cqc(struct udma_dev *udma_dev, struct udma_jfc *udma_jfc) return ret; } +static void store_jfc_id(struct udma_dev *udma_dev, struct udma_jfc *udma_jfc) +{ + struct jfc_list *jfc_new; + struct jfc_list *jfc_now; + unsigned long flags; + spinlock_t *lock; + int ret; + int i; + + ret = udma_find_dfx_dev(udma_dev, &i); + if (ret) + return; + + jfc_new = kzalloc(sizeof(struct jfc_list), GFP_KERNEL); + if (jfc_new == NULL) + return; + + lock = &g_udma_dfx_list[i].dfx->jfc_list->node_lock; + spin_lock_irqsave(lock, flags); + list_for_each_entry(jfc_now, + &g_udma_dfx_list[i].dfx->jfc_list->node, node) { + if (jfc_now->jfc_id == udma_jfc->cqn) + goto found; + } + + jfc_new->jfc_id = udma_jfc->cqn; + list_add(&jfc_new->node, &g_udma_dfx_list[i].dfx->jfc_list->node); + spin_unlock_irqrestore(lock, flags); + + return; + +found: + spin_unlock_irqrestore(lock, flags); + kfree(jfc_new); +} + +static void delete_jfc_id(struct udma_dev *udma_dev, struct udma_jfc *udma_jfc) +{ + struct jfc_list *jfc_now, *jfc_tmp; + unsigned long flags; + spinlock_t *lock; + int ret; + int i; + + ret = udma_find_dfx_dev(udma_dev, &i); + if (ret) + return; + + lock = &g_udma_dfx_list[i].dfx->jfc_list->node_lock; + spin_lock_irqsave(lock, flags); + list_for_each_entry_safe(jfc_now, jfc_tmp, + &g_udma_dfx_list[i].dfx->jfc_list->node, + node) { + if (jfc_now->jfc_id == udma_jfc->cqn) { + list_del(&jfc_now->node); + kfree(jfc_now); + spin_unlock_irqrestore(lock, flags); + return; + } + } + spin_unlock_irqrestore(lock, flags); +} + static void free_jfc_cqc(struct udma_dev *udma_dev, struct udma_jfc *udma_jfc) { struct udma_jfc_table *jfc_table = &udma_dev->jfc_table; @@ -528,6 +592,9 @@ struct ubcore_jfc *udma_create_jfc(struct ubcore_device *dev, const struct ubcor if (ret) goto err_jfc_id; + if (dfx_switch) + store_jfc_id(udma_dev, udma_jfc); + return &udma_jfc->ubcore_jfc; err_jfc_id: @@ -546,6 +613,10 @@ int udma_destroy_jfc(struct ubcore_jfc *jfc) struct udma_jfc *udma_jfc = to_udma_jfc(jfc); free_jfc_cqc(udma_dev, udma_jfc); + + if (dfx_switch) + delete_jfc_id(udma_dev, udma_jfc); + free_jfc_id(udma_dev, udma_jfc); free_jfc_buf(udma_dev, udma_jfc); kfree(udma_jfc); diff --git a/drivers/ub/hw/hns3/hns3_udma_jfr.c b/drivers/ub/hw/hns3/hns3_udma_jfr.c index 59801f84ce7d..9deb39423bec 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jfr.c +++ b/drivers/ub/hw/hns3/hns3_udma_jfr.c @@ -18,6 +18,7 @@ #include "hns3_udma_cmd.h" #include "hns3_udma_db.h" #include "hns3_udma_jfc.h" +#include "hns3_udma_dfx.h" #include "hns3_udma_jfr.h" static int init_jfr_cfg(struct udma_dev *dev, struct udma_jfr *jfr, @@ -363,6 +364,73 @@ static int udma_hw_destroy_srq(struct udma_dev *dev, uint64_t jfrn) return udma_cmd_mbox(dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); } +static void store_jfr_id(struct udma_dev *dev, struct udma_jfr *jfr) +{ + struct jfr_list *jfr_new; + struct jfr_list *jfr_now; + unsigned long flags; + spinlock_t *lock; + int ret; + int i; + + ret = udma_find_dfx_dev(dev, &i); + if (ret) + return; + + jfr_new = kzalloc(sizeof(struct jfr_list), GFP_KERNEL); + if (jfr_new == NULL) + return; + + lock = &g_udma_dfx_list[i].dfx->jfr_list->node_lock; + spin_lock_irqsave(lock, flags); + list_for_each_entry(jfr_now, + &g_udma_dfx_list[i].dfx->jfr_list->node, + node) { + if (jfr_now->jfr_id == jfr->jfrn) { + jfr_now->jfc_id = jfr->ubcore_jfr.jfr_cfg.jfc->id; + goto found; + } + } + + jfr_new->jfr_id = jfr->jfrn; + jfr_new->jfc_id = jfr->ubcore_jfr.jfr_cfg.jfc->id; + list_add(&jfr_new->node, &g_udma_dfx_list[i].dfx->jfr_list->node); + spin_unlock_irqrestore(lock, flags); + + return; + +found: + spin_unlock_irqrestore(lock, flags); + kfree(jfr_new); +} + +static void delete_jfr_id(struct udma_dev *dev, struct udma_jfr *jfr) +{ + struct jfr_list *jfr_now, *jfr_tmp; + unsigned long flags; + spinlock_t *lock; + int ret; + int i; + + ret = udma_find_dfx_dev(dev, &i); + if (ret) + return; + + lock = &g_udma_dfx_list[i].dfx->jfr_list->node_lock; + spin_lock_irqsave(lock, flags); + list_for_each_entry_safe(jfr_now, jfr_tmp, + &g_udma_dfx_list[i].dfx->jfr_list->node, + node) { + if (jfr_now->jfr_id == jfr->jfrn) { + list_del(&jfr_now->node); + kfree(jfr_now); + spin_unlock_irqrestore(lock, flags); + return; + } + } + spin_unlock_irqrestore(lock, flags); +} + static void free_jfrc(struct udma_dev *dev, uint32_t jfrn) { struct udma_jfr_table *jfr_table = &dev->jfr_table; @@ -495,6 +563,10 @@ struct ubcore_jfr *udma_create_jfr(struct ubcore_device *dev, const struct ubcor if (ret) goto err_alloc_jfrc; } + + if (dfx_switch) + store_jfr_id(udma_dev, jfr); + return &jfr->ubcore_jfr; err_alloc_jfrc: @@ -516,6 +588,10 @@ int udma_destroy_jfr(struct ubcore_jfr *jfr) if (udma_jfr->um_qp) destroy_jfr_um_qp(dev, udma_jfr); clean_jetty_x_qpn_bitmap(&udma_jfr->qpn_map); + + if (dfx_switch) + delete_jfr_id(dev, udma_jfr); + free_jfrc(dev, udma_jfr->jfrn); free_jfr_buf(dev, udma_jfr); kfree(jfr); diff --git a/drivers/ub/hw/hns3/hns3_udma_jfs.c b/drivers/ub/hw/hns3/hns3_udma_jfs.c index 7fba63d6f46d..9e4944c5d732 100644 --- a/drivers/ub/hw/hns3/hns3_udma_jfs.c +++ b/drivers/ub/hw/hns3/hns3_udma_jfs.c @@ -18,6 +18,7 @@ #include "hns3_udma_hem.h" #include "hns3_udma_tp.h" #include "hns3_udma_jfc.h" +#include "hns3_udma_dfx.h" #include "hns3_udma_jfs.h" static int init_jfs_cfg(struct udma_dev *dev, struct udma_jfs *jfs, @@ -187,6 +188,76 @@ static int alloc_jfs_id(struct udma_dev *udma_dev, struct udma_jfs *jfs) return ret; } +static void store_jfs_id(struct udma_dev *udma_dev, struct udma_jfs *jfs) +{ + struct jfs_list *jfs_new; + struct jfs_list *jfs_now; + unsigned long flags; + spinlock_t *lock; + int ret; + int i; + + ret = udma_find_dfx_dev(udma_dev, &i); + if (ret) + return; + + jfs_new = kzalloc(sizeof(struct jfs_list), GFP_KERNEL); + if (jfs_new == NULL) + return; + + lock = &g_udma_dfx_list[i].dfx->jfs_list->node_lock; + spin_lock_irqsave(lock, flags); + list_for_each_entry(jfs_now, + &g_udma_dfx_list[i].dfx->jfs_list->node, node) { + if (jfs_now->jfs_id == jfs->jfs_id) { + jfs_now->depth = jfs->ubcore_jfs.jfs_cfg.depth; + jfs_now->pri = jfs->ubcore_jfs.jfs_cfg.priority; + jfs_now->jfc_id = jfs->ubcore_jfs.jfs_cfg.jfc->id; + goto found; + } + } + + jfs_new->jfs_id = jfs->jfs_id; + jfs_new->depth = jfs->ubcore_jfs.jfs_cfg.depth; + jfs_new->pri = jfs->ubcore_jfs.jfs_cfg.priority; + jfs_new->jfc_id = jfs->ubcore_jfs.jfs_cfg.jfc->id; + list_add(&jfs_new->node, &g_udma_dfx_list[i].dfx->jfs_list->node); + spin_unlock_irqrestore(lock, flags); + + return; + +found: + spin_unlock_irqrestore(lock, flags); + kfree(jfs_new); +} + +static void delete_jfs_id(struct udma_dev *udma_dev, struct udma_jfs *jfs) +{ + struct jfs_list *jfs_now, *jfs_tmp; + unsigned long flags; + spinlock_t *lock; + int ret; + int i; + + ret = udma_find_dfx_dev(udma_dev, &i); + if (ret) + return; + + lock = &g_udma_dfx_list[i].dfx->jfs_list->node_lock; + spin_lock_irqsave(lock, flags); + list_for_each_entry_safe(jfs_now, jfs_tmp, + &g_udma_dfx_list[i].dfx->jfs_list->node, + node) { + if (jfs_now->jfs_id == jfs->jfs_id) { + list_del(&jfs_now->node); + kfree(jfs_now); + spin_unlock_irqrestore(lock, flags); + return; + } + } + spin_unlock_irqrestore(lock, flags); +} + static void free_jfs_id(struct udma_dev *udma_dev, struct udma_jfs *jfs) { struct udma_jfs_table *jfs_table = &udma_dev->jfs_table; @@ -225,6 +296,9 @@ struct ubcore_jfs *udma_create_jfs(struct ubcore_device *dev, const struct ubcor goto err_alloc_jfs_buf; } + if (dfx_switch) + store_jfs_id(udma_dev, jfs); + return &jfs->ubcore_jfs; err_alloc_jfs_buf: @@ -248,6 +322,10 @@ int udma_destroy_jfs(struct ubcore_jfs *jfs) ret = destroy_jfs_qp(udma_dev, udma_jfs); clean_jetty_x_qpn_bitmap(&udma_jfs->qpn_map); + + if (dfx_switch) + delete_jfs_id(udma_dev, udma_jfs); + free_jfs_id(udma_dev, udma_jfs); kfree(udma_jfs); diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c index 53f5d237f66b..3daa2e99bbdc 100644 --- a/drivers/ub/hw/hns3/hns3_udma_main.c +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -28,6 +28,7 @@ #include "hns3_udma_jetty.h" #include "hns3_udma_dca.h" #include "hns3_udma_cmd.h" +#include "hns3_udma_dfx.h" static int udma_set_eid(struct ubcore_device *dev, union ubcore_eid eid) { @@ -835,6 +836,7 @@ static struct ubcore_ops g_udma_dev_ops = { .set_eid = udma_set_eid, .query_device_attr = udma_query_device_attr, .query_device_status = udma_query_device_status, + .query_res = udma_query_res, .alloc_ucontext = udma_alloc_ucontext, .free_ucontext = udma_free_ucontext, .mmap = udma_mmap, diff --git a/drivers/ub/hw/hns3/hns3_udma_segment.c b/drivers/ub/hw/hns3/hns3_udma_segment.c index acb9c2ee9715..817bea1a83fd 100644 --- a/drivers/ub/hw/hns3/hns3_udma_segment.c +++ b/drivers/ub/hw/hns3/hns3_udma_segment.c @@ -17,6 +17,7 @@ #include "hns3_udma_abi.h" #include "hns3_udma_hem.h" #include "hns3_udma_cmd.h" +#include "hns3_udma_dfx.h" #include "hns3_udma_segment.h" static uint32_t hw_index_to_key(int ind) @@ -233,6 +234,76 @@ static void free_seg_key(struct udma_dev *udma_dev, struct udma_seg *seg) ida_free(&udma_dev->seg_table.seg_ida.ida, (int)obj); } +static void store_seg_id(struct udma_dev *udma_dev, struct udma_seg *seg) +{ + struct seg_list *seg_new; + struct seg_list *seg_now; + unsigned long flags; + spinlock_t *lock; + int ret; + int i; + + ret = udma_find_dfx_dev(udma_dev, &i); + if (ret) + return; + + seg_new = kzalloc(sizeof(struct seg_list), GFP_KERNEL); + if (seg_new == NULL) + return; + + lock = &g_udma_dfx_list[i].dfx->seg_list->node_lock; + spin_lock_irqsave(lock, flags); + list_for_each_entry(seg_now, + &g_udma_dfx_list[i].dfx->seg_list->node, node) { + if (seg_now->key_id == seg->key) { + seg_now->pd = seg->pd; + seg_now->iova = seg->iova; + seg_now->len = seg->size; + goto found; + } + } + + seg_new->pd = seg->pd; + seg_new->iova = seg->iova; + seg_new->len = seg->size; + seg_new->key_id = seg->key; + list_add(&seg_new->node, &g_udma_dfx_list[i].dfx->seg_list->node); + spin_unlock_irqrestore(lock, flags); + + return; + +found: + spin_unlock_irqrestore(lock, flags); + kfree(seg_new); +} + +static void delete_seg_id(struct udma_dev *udma_dev, struct udma_seg *seg) +{ + struct seg_list *seg_now, *seg_tmp; + unsigned long flags; + spinlock_t *lock; + int ret; + int i; + + ret = udma_find_dfx_dev(udma_dev, &i); + if (ret) + return; + + lock = &g_udma_dfx_list[i].dfx->seg_list->node_lock; + spin_lock_irqsave(lock, flags); + list_for_each_entry_safe(seg_now, seg_tmp, + &g_udma_dfx_list[i].dfx->seg_list->node, + node) { + if (seg_now->key_id == seg->key) { + list_del(&seg_now->node); + kfree(seg_now); + spin_unlock_irqrestore(lock, flags); + return; + } + } + spin_unlock_irqrestore(lock, flags); +} + struct ubcore_target_seg *udma_register_seg(struct ubcore_device *dev, const struct ubcore_seg_cfg *cfg, struct ubcore_udata *udata) @@ -272,6 +343,8 @@ struct ubcore_target_seg *udma_register_seg(struct ubcore_device *dev, seg->enabled = 1; seg->ubcore_seg.seg.key_id = seg->key; + if (dfx_switch) + store_seg_id(udma_dev, seg); return &seg->ubcore_seg; @@ -306,6 +379,9 @@ int udma_unregister_seg(struct ubcore_target_seg *seg) struct udma_dev *udma_dev = to_udma_dev(seg->ub_dev); struct udma_seg *udma_seg = to_udma_seg(seg); + if (dfx_switch) + delete_seg_id(udma_dev, udma_seg); + udma_seg_free(udma_dev, udma_seg); kfree(udma_seg); diff --git a/drivers/ub/hw/hns3/hns3_udma_tp.c b/drivers/ub/hw/hns3/hns3_udma_tp.c index 9f118710779d..ef2eb335fb87 100644 --- a/drivers/ub/hw/hns3/hns3_udma_tp.c +++ b/drivers/ub/hw/hns3/hns3_udma_tp.c @@ -20,6 +20,7 @@ #include "hns3_udma_jetty.h" #include "hns3_udma_hem.h" #include "hns3_udma_dca.h" +#include "hns3_udma_dfx.h" #include "hns3_udma_tp.h" struct udma_qp *get_qp(struct udma_dev *udma_device, uint32_t qpn) { @@ -168,6 +169,70 @@ void *udma_erase_tp(struct udma_tp *udma_tp) return udma_tp; } +static void store_tpn(struct udma_dev *udma_device, struct udma_tp *tp) +{ + struct tpn_list *tpn_new; + struct tpn_list *tpn_now; + unsigned long flags; + spinlock_t *lock; + int ret; + int i; + + ret = udma_find_dfx_dev(udma_device, &i); + if (ret) + return; + + tpn_new = kzalloc(sizeof(struct tpn_list), GFP_KERNEL); + if (tpn_new == NULL) + return; + + lock = &g_udma_dfx_list[i].dfx->tpn_list->node_lock; + spin_lock_irqsave(lock, flags); + list_for_each_entry(tpn_now, + &g_udma_dfx_list[i].dfx->tpn_list->node, node) { + if (tpn_now->tpn == tp->ubcore_tp.tpn) + goto found; + } + + tpn_new->tpn = tp->ubcore_tp.tpn; + list_add(&tpn_new->node, &g_udma_dfx_list[i].dfx->tpn_list->node); + spin_unlock_irqrestore(lock, flags); + + return; + +found: + spin_unlock_irqrestore(lock, flags); + kfree(tpn_new); +} + +static void delete_tpn(struct udma_dev *udma_device, struct ubcore_tp *tp) +{ + struct tpn_list *tpn_now, *tpn_tmp; + unsigned long flags; + spinlock_t *lock; + int ret; + int i; + + ret = udma_find_dfx_dev(udma_device, &i); + if (ret) + return; + + lock = &g_udma_dfx_list[i].dfx->tpn_list->node_lock; + spin_lock_irqsave(lock, flags); + list_for_each_entry_safe(tpn_now, tpn_tmp, + &g_udma_dfx_list[i].dfx->tpn_list->node, + node) { + if (tpn_now->tpn == tp->tpn) { + list_del(&tpn_now->node); + kfree(tpn_now); + spin_unlock_irqrestore(lock, flags); + return; + } + } + spin_unlock_irqrestore(lock, flags); +} + + int udma_destroy_tp(struct ubcore_tp *tp) { struct udma_dev *udma_device = to_udma_dev(tp->ub_dev); @@ -184,6 +249,9 @@ int udma_destroy_tp(struct ubcore_tp *tp) return 0; } + if (dfx_switch) + delete_tpn(udma_device, tp); + qp = &udma_tp->qp; curr_state = to_udma_qp_state(tp->state); ubcore_attr_mask.value = 0; @@ -412,6 +480,9 @@ struct ubcore_tp *udma_create_tp(struct ubcore_device *dev, const struct ubcore_ if (ret || fail_ret_tp) goto failed_create_qp; + if (dfx_switch) + store_tpn(udma_dev, tp); + return &tp->ubcore_tp; failed_create_qp: -- Gitee From 7817547088b22219d845379e662f04326e97eac4 Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Tue, 19 Sep 2023 21:12:12 +0800 Subject: [PATCH 42/42] hns3 udma: support dfx function of query stats of TX and RX packets. driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I85R2F CVE: NA --------------------------------------------------------------------- This patch supports dfx function of query stats of TX and RX packets. Signed-off-by: Yu Zhang --- drivers/ub/hw/hns3/hns3_udma_hw.h | 22 +++++++++++++ drivers/ub/hw/hns3/hns3_udma_main.c | 50 +++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.h b/drivers/ub/hw/hns3/hns3_udma_hw.h index cb970fb99fab..17e1c2750cd8 100644 --- a/drivers/ub/hw/hns3/hns3_udma_hw.h +++ b/drivers/ub/hw/hns3/hns3_udma_hw.h @@ -134,6 +134,11 @@ struct udma_query_oor_cmq { #define SPEED_100G 100000 #define SPEED_200G 200000 +#define UDMA_QUERY_COUNTER 8 +#define UDMA_QX_RESP 1 +#define UDMA_TX_RESP 3 +#define UDMA_TX_ERR_RESP 4 + /* Fields of UDMA_OPC_EXT_CFG */ #define EXT_CFG_VF_ID CMQ_REQ_FIELD_LOC(31, 0) #define EXT_CFG_QP_PI_INDEX CMQ_REQ_FIELD_LOC(45, 32) @@ -242,6 +247,7 @@ enum udma_opcode_type { UDMA_OPC_CFG_GMV_TBL = 0xA140, UDMA_OPC_CFG_POE_ADDR = 0x801B, UDMA_OPC_CFG_POE_ATTR = 0x801C, + UDMA_OPC_QUERY_COUNTER = 0x8206, UDMA_OPC_QUERY_PORT_INFO = 0x7104, }; @@ -514,4 +520,20 @@ struct udma_port_info_cmq { uint8_t rsv[19]; }; +struct udma_rx_cnt_cmd_data { + uint64_t rsv; + uint64_t pkt_rx_cnt; + uint64_t err_pkt_rx_cnt; +}; + +struct udma_tx_cnt_cmd_data { + uint64_t rsv[2]; + uint64_t pkt_tx_cnt; +}; + +struct udma_tx_err_cnt_cmd_data { + uint64_t err_pkt_tx_cnt; + uint64_t rsv[2]; +}; + #endif /* _UDMA_HW_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c index 3daa2e99bbdc..ab105f589a64 100644 --- a/drivers/ub/hw/hns3/hns3_udma_main.c +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -260,6 +260,55 @@ static int udma_mmap(struct ubcore_ucontext *uctx, struct vm_area_struct *vma) return 0; } + +static int udma_query_stats(const struct ubcore_device *dev, struct ubcore_stats_key *key, + struct ubcore_stats_val *val) +{ + struct ubcore_stats_com_val *com_val = (struct ubcore_stats_com_val *)val->addr; + struct udma_cmq_desc desc[UDMA_QUERY_COUNTER]; + struct udma_dev *udma_dev = to_udma_dev(dev); + struct udma_tx_err_cnt_cmd_data *resp_tx_err; + struct udma_rx_cnt_cmd_data *resp_rx; + struct udma_tx_cnt_cmd_data *resp_tx; + int ret; + int i; + + if (val->len != sizeof(struct ubcore_stats_com_val)) { + dev_err(udma_dev->dev, "The val len is err.\n"); + return -EINVAL; + } + + for (i = 0; i < UDMA_QUERY_COUNTER; i++) { + udma_cmq_setup_basic_desc(&desc[i], UDMA_OPC_QUERY_COUNTER, true); + if (i < (UDMA_QUERY_COUNTER - 1)) + desc[i].flag |= cpu_to_le16(UDMA_CMD_FLAG_NEXT); + else + desc[i].flag &= ~cpu_to_le16(UDMA_CMD_FLAG_NEXT); + } + + ret = udma_cmq_send(udma_dev, desc, UDMA_QUERY_COUNTER); + if (ret) { + dev_err(udma_dev->dev, "Failed to query stats, ret = %d.\n", ret); + return ret; + } + + resp_rx = (struct udma_rx_cnt_cmd_data *)desc[UDMA_QX_RESP].data; + resp_tx = (struct udma_tx_cnt_cmd_data *)desc[UDMA_TX_RESP].data; + resp_tx_err = (struct udma_tx_err_cnt_cmd_data *)desc[UDMA_TX_ERR_RESP].data; + + com_val->tx_pkt = resp_tx->pkt_tx_cnt; + com_val->tx_pkt_err = resp_tx_err->err_pkt_tx_cnt; + + com_val->rx_pkt = resp_rx->pkt_rx_cnt; + com_val->rx_pkt_err = resp_rx->err_pkt_rx_cnt; + + /* tx_bytes and rx_bytes are not support now */ + com_val->tx_bytes = 0; + com_val->rx_bytes = 0; + + return ret; +} + static uint16_t query_congest_alg(uint8_t udma_cc_caps) { uint16_t ubcore_cc_alg = 0; @@ -862,6 +911,7 @@ static struct ubcore_ops g_udma_dev_ops = { .modify_tp = udma_modify_tp, .destroy_tp = udma_destroy_tp, .user_ctl = udma_user_ctl, + .query_stats = udma_query_stats, }; static void udma_cleanup_uar_table(struct udma_dev *dev) -- Gitee