diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig index 0c3e2a79be42ef011e747270ea56df44ca30ce76..1e80e621ba28f48cab5d1d907c4f5c6b4570ec19 100644 --- a/arch/arm64/configs/openeuler_defconfig +++ b/arch/arm64/configs/openeuler_defconfig @@ -6209,6 +6209,7 @@ CONFIG_ROH=m CONFIG_ROH_HNS=m CONFIG_UB=m CONFIG_UB_URMA=m +CONFIG_UB_UDMA_HNS3=m # end of Device Drivers # diff --git a/drivers/ub/Kconfig b/drivers/ub/Kconfig index 46651e1752b5fee39e69efe99c12898968718c0b..74640ce3c933d9ebc929894d152e9cc803e49b44 100644 --- a/drivers/ub/Kconfig +++ b/drivers/ub/Kconfig @@ -23,4 +23,5 @@ config UB_URMA need liburma from umdk . +source "drivers/ub/hw/hns3/Kconfig" endif # UB diff --git a/drivers/ub/Makefile b/drivers/ub/Makefile index 9cd43d23bffa2175c2c5e2b5b91003cbeefccbc0..d60ee553690b0539b79a96e0994b1e51f894a94f 100644 --- a/drivers/ub/Makefile +++ b/drivers/ub/Makefile @@ -4,3 +4,4 @@ # obj-$(CONFIG_UB) += urma/ +obj-$(CONFIG_UB_UDMA_HNS3) += hw/hns3/ diff --git a/drivers/ub/hw/hns3/Kconfig b/drivers/ub/hw/hns3/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..82ca7b9106739808fdd5978063001be780901343 --- /dev/null +++ b/drivers/ub/hw/hns3/Kconfig @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# UDMA_HNS configuration +# + +config UB_UDMA_HNS3 + default n + tristate "HNS3 UB UDMA Driver" + depends on UB && UB_URMA + depends on ARM64 + depends on UBL && PCI + help + This is a UB_UDMA_HNS3 driver for the Hisilicon UDMA engine. + This driver depands on ubcore and uburma. + This driver depands on UBL(UB link). The UDMA_HNS3 is a PCI device. + To compile UB_UDMA driver as module, choose M here. + module will be called hns3_udma. diff --git a/drivers/ub/hw/hns3/Makefile b/drivers/ub/hw/hns3/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..533cadbd826354d5e616982c18eb38624552bab0 --- /dev/null +++ b/drivers/ub/hw/hns3/Makefile @@ -0,0 +1,17 @@ +# +## Makefile for the Huawei UDMA Linux driver. +# +## + +MODULE_NAME := hns3_udma +ccflags-y += -I$(srctree)/drivers/net/ethernet/hisilicon/hns3/ \ + -I$(srctree)/drivers/ub/urma/ + +$(MODULE_NAME)-objs := hns3_udma_hw.o hns3_udma_main.o hns3_udma_cmd.o \ + hns3_udma_hem.o hns3_udma_qp.o hns3_udma_eq.o \ + hns3_udma_db.o hns3_udma_jfc.o hns3_udma_jfr.o \ + hns3_udma_segment.o hns3_udma_tp.o hns3_udma_jfs.o \ + hns3_udma_jetty.o hns3_udma_sysfs.o hns3_udma_dca.o \ + hns3_udma_dfx.o + +obj-$(CONFIG_UB_UDMA_HNS3) := hns3_udma.o diff --git a/drivers/ub/hw/hns3/hns3_udma_abi.h b/drivers/ub/hw/hns3/hns3_udma_abi.h new file mode 100644 index 0000000000000000000000000000000000000000..29b66ba20179b7f2ef4ac9dbf7f4b616d5af2c93 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_abi.h @@ -0,0 +1,256 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_ABI_H +#define _UDMA_ABI_H + +#include + +#define MAP_COMMAND_MASK 0xff +#define MAP_INDEX_MASK 0xffffff +#define MAP_INDEX_SHIFT 8 +#define UDMA_DWQE_PAGE_SIZE 65536 +#define UDMA_JETTY_X_PREFIX_BIT_NUM 2 +#define UDMA_JFS_QPN_PREFIX 0x0 +#define UDMA_JFR_QPN_PREFIX 0x1 +#define UDMA_JETTY_QPN_PREFIX 0x2 +#define UDMA_ADDR_4K_MASK 0xfffUL +#define URMA_SEG_ACCESS_GUARD (1UL << 5) +#define UDMA_DCA_ATTACH_FLAGS_NEW_BUFFER BIT(0) +#define UDMA_DCA_INVALID_DCA_NUM ~0U + +enum { + UDMA_MMAP_UAR_PAGE, + UDMA_MMAP_DWQE_PAGE, + UDMA_MMAP_DCA_PAGE, + UDMA_MMAP_RESET_PAGE, + UDMA_MMAP_TYPE_DCA +}; + +enum udma_jfc_init_attr_mask { + UDMA_JFC_NOTIFY_OR_POE_CREATE_FLAGS = 1 << 0, +}; + +enum udma_jfc_create_flags { + UDMA_JFC_CREATE_ENABLE_POE_MODE = 1 << 0, + UDMA_JFC_CREATE_ENABLE_NOTIFY = 1 << 1, +}; + +enum udma_jfc_notify_mode { + UDMA_JFC_NOTIFY_MODE_64B_ALIGN, + UDMA_JFC_NOTIFY_MODE_4B_ALIGN, + UDMA_JFC_NOTIFY_MODE_DDR_64B_ALIGN, + UDMA_JFC_NOTIFY_MODE_DDR_4B_ALIGN, +}; + +struct udma_create_jfr_ucmd { + uint64_t buf_addr; + uint64_t idx_addr; + uint64_t db_addr; +}; + +enum udma_jfr_cap_flags { + UDMA_JFR_CAP_RECORD_DB = 1 << 0, +}; + +struct udma_create_jfr_resp { + uint32_t jfr_caps; +}; + +struct udma_jfc_attr_ex { + uint64_t jfc_ex_mask; /* Use enum udma_jfc_init_attr_mask */ + uint64_t create_flags; /* Use enum udma_jfc_create_flags */ + uint8_t poe_channel; /* poe channel to use */ + uint64_t notify_addr; + uint8_t notify_mode; /* Use enum udma_jfc_notify_mode */ +}; + +struct udma_create_jfc_ucmd { + uint64_t buf_addr; + uint64_t db_addr; + struct udma_jfc_attr_ex jfc_attr_ex; +}; + +enum udma_jfc_cap_flags { + UDMA_JFC_CAP_RECORD_DB = 1 << 0, +}; + +struct udma_create_jfc_resp { + uint32_t jfc_caps; +}; + +struct udma_create_tp_ucmd { + bool is_jetty; + union { + uint32_t jfs_id; + uint32_t jetty_id; + } ini_id; + union { + uint32_t jfr_id; + uint32_t jetty_id; + } tgt_id; + /* used for create_qp */ + uint64_t buf_addr; + uint64_t db_addr; + uint64_t sdb_addr; +}; + +struct udma_create_jetty_ucmd { + struct udma_create_tp_ucmd create_tp_ucmd; + uint32_t jfr_id; + uint64_t buf_addr; + uint64_t sdb_addr; +}; + +enum udma_qp_cap_flags { + UDMA_QP_CAP_RQ_RECORD_DB = 1 << 0, + UDMA_QP_CAP_SQ_RECORD_DB = 1 << 1, + UDMA_QP_CAP_OWNER_DB = 1 << 2, + UDMA_QP_CAP_DYNAMIC_CTX_ATTACH = 1 << 4, + UDMA_QP_CAP_DIRECT_WQE = 1 << 5, +}; + +struct udp_srcport { + bool um_spray_en; + uint16_t um_data_udp_start; + uint8_t um_udp_range; +}; + +struct udma_create_tp_resp { + uint64_t cap_flags; + uint32_t qpn; + uint32_t path_mtu; + uint8_t priority; + struct udp_srcport um_srcport; +}; + +struct udma_create_jetty_resp { + struct udma_create_tp_resp create_tp_resp; +}; + +struct udma_create_jfs_ucmd { + struct udma_create_tp_ucmd create_tp_ucmd; +}; + +struct udma_create_jfs_resp { + struct udma_create_tp_resp create_tp_resp; +}; + +struct udma_create_ctx_ucmd { + uint32_t comp; + uint32_t dca_max_qps; + uint32_t dca_unit_size; +}; + +enum udma_context_comp_mask { + UDMA_CONTEXT_MASK_DCA_PRIME_QPS = 1 << 0, + UDMA_CONTEXT_MASK_DCA_UNIT_SIZE = 1 << 1, + UDMA_CONTEXT_MASK_DCA_MAX_SIZE = 1 << 2, + UDMA_CONTEXT_MASK_DCA_MIN_SIZE = 1 << 3, +}; + +struct udma_create_ctx_resp { + uint32_t num_comp_vectors; + uint32_t num_qps_shift; + uint32_t num_jfs_shift; + uint32_t num_jfr_shift; + uint32_t num_jetty_shift; + uint32_t max_jfc_cqe; + uint32_t cqe_size; + uint32_t max_jfr_wr; + uint32_t max_jfr_sge; + uint32_t max_jfs_wr; + uint32_t max_jfs_sge; + uint32_t poe_ch_num; + uint64_t db_addr; + uint32_t dca_qps; + uint32_t dca_mmap_size; + uint32_t dca_mode; +}; + +struct flush_cqe_param { + uint32_t qpn; + uint32_t sq_producer_idx; +}; + +struct udma_poe_info { + uint8_t en; + uint8_t poe_channel; + uint64_t poe_addr; +}; + +struct udma_dca_reg_attr { + uintptr_t key; + uintptr_t addr; + uint32_t size; +}; + +struct udma_dca_dereg_attr { + uintptr_t free_key; + struct dca_mem *mem; +}; + +struct udma_dca_shrink_attr { + uint64_t reserved_size; +}; + +struct udma_dca_shrink_resp { + struct dca_mem *mem; + uintptr_t free_key; + uint32_t free_mems; +}; + +struct udma_dca_attach_attr { + uint64_t qpn; + uint32_t sq_offset; + uint32_t sge_offset; +}; + +struct udma_dca_attach_resp { + uint32_t alloc_flags; + uint32_t alloc_pages; + uint32_t dcan; +}; + +struct udma_dca_detach_attr { + uint64_t qpn; + uint32_t sq_idx; +}; + +struct udma_dca_query_attr { + uint64_t qpn; + uint32_t page_idx; +}; + +struct udma_dca_query_resp { + uintptr_t mem_key; + uint32_t mem_ofs; + uint32_t page_count; +}; + +enum udma_user_ctl_handlers { + UDMA_USER_CTL_FLUSH_CQE, + UDMA_CONFIG_POE_CHANNEL, + UDMA_QUERY_POE_CHANNEL, + UDMA_DCA_MEM_REG, + UDMA_DCA_MEM_DEREG, + UDMA_DCA_MEM_SHRINK, + UDMA_DCA_MEM_ATTACH, + UDMA_DCA_MEM_DETACH, + UDMA_DCA_MEM_QUERY, + UDMA_OPCODE_NUM, +}; + +#endif /* _UDMA_ABI_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_cmd.c b/drivers/ub/hw/hns3/hns3_udma_cmd.c new file mode 100644 index 0000000000000000000000000000000000000000..7271ab5cc002c1264f732224ce5727e8bdade9c6 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_cmd.c @@ -0,0 +1,604 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include +#include +#include +#include "hnae3.h" +#include "hns3_udma_common.h" +#include "hns3_udma_cmd.h" + +int udma_cmd_init(struct udma_dev *udma_dev) +{ + sema_init(&udma_dev->cmd.poll_sem, 1); + udma_dev->cmd.use_events = 0; + udma_dev->cmd.max_cmds = CMD_MAX_NUM; + udma_dev->cmd.pool = dma_pool_create("udma_cmd", udma_dev->dev, + UDMA_MAILBOX_SIZE, + UDMA_MAILBOX_SIZE, 0); + if (!udma_dev->cmd.pool) + return -ENOMEM; + + init_rwsem(&udma_dev->cmd.udma_mb_rwsem); + + return 0; +} + +void udma_cmd_cleanup(struct udma_dev *udma_dev) +{ + down_write(&udma_dev->cmd.udma_mb_rwsem); + dma_pool_destroy(udma_dev->cmd.pool); + up_write(&udma_dev->cmd.udma_mb_rwsem); +} + +int udma_cmd_use_events(struct udma_dev *udma_dev) +{ + struct udma_cmdq *udma_cmd = &udma_dev->cmd; + int i; + + udma_cmd->context = (struct udma_cmd_context *) + kcalloc(udma_cmd->max_cmds, sizeof(*udma_cmd->context), + GFP_KERNEL); + if (!udma_cmd->context) + return -ENOMEM; + + for (i = 0; i < udma_cmd->max_cmds; ++i) { + udma_cmd->context[i].token = i; + udma_cmd->context[i].next = i + 1; + init_completion(&udma_cmd->context[i].done); + } + udma_cmd->context[udma_cmd->max_cmds - 1].next = 0; + udma_cmd->free_head = 0; + + sema_init(&udma_cmd->event_sem, udma_cmd->max_cmds); + spin_lock_init(&udma_cmd->ctx_lock); + + udma_cmd->use_events = 1; + + return 0; +} + +void udma_cmd_use_polling(struct udma_dev *udma_dev) +{ + struct udma_cmdq *udma_cmd = &udma_dev->cmd; + + kfree(udma_cmd->context); + udma_cmd->use_events = 0; +} + +struct udma_cmd_mailbox *udma_alloc_cmd_mailbox(struct udma_dev *dev) +{ + struct udma_cmd_mailbox *mailbox; + + mailbox = kzalloc(sizeof(*mailbox), GFP_KERNEL); + if (!mailbox) + return ERR_PTR(-ENOMEM); + + down_read(&dev->cmd.udma_mb_rwsem); + mailbox->buf = + dma_pool_zalloc(dev->cmd.pool, GFP_KERNEL, &mailbox->dma); + if (!mailbox->buf) { + up_read(&dev->cmd.udma_mb_rwsem); + + kfree(mailbox); + return ERR_PTR(-ENOMEM); + } + + return mailbox; +} + +void udma_free_cmd_mailbox(struct udma_dev *dev, + struct udma_cmd_mailbox *mailbox) +{ + if (!mailbox) + return; + + dma_pool_free(dev->cmd.pool, mailbox->buf, mailbox->dma); + + up_read(&dev->cmd.udma_mb_rwsem); + + kfree(mailbox); +} + +static uint32_t udma_cmd_hw_reseted(struct udma_dev *dev, + uint64_t instance_stage, + uint64_t reset_stage) +{ + /* When hardware reset has been completed once or more, we should stop + * sending mailbox&cmq&doorbell to hardware. If now in .init_instance() + * function, we should exit with error. If now at HNAE3_INIT_CLIENT + * stage of soft reset process, we should exit with error, and then + * HNAE3_INIT_CLIENT related process can rollback the operation like + * notifing hardware to free resources, HNAE3_INIT_CLIENT related + * process will exit with error to notify NIC driver to reschedule soft + * reset process once again. + */ + dev->is_reset = true; + dev->dis_db = true; + + if (reset_stage == UDMA_STATE_RST_INIT || + instance_stage == UDMA_STATE_INIT) + return CMD_RST_PRC_EBUSY; + + return CMD_RST_PRC_SUCCESS; +} + +static uint32_t udma_cmd_hw_resetting(struct udma_dev *dev, + uint64_t instance_stage, + uint64_t reset_stage) +{ +#define HW_RESET_TIMEOUT_US 1000000 +#define HW_RESET_DELAY_US 1 + struct udma_priv *priv = (struct udma_priv *)dev->priv; + struct hnae3_handle *handle = priv->handle; + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + uint64_t val; + int ret; + + /* When hardware reset is detected, we should stop sending mailbox&cmq& + * doorbell to hardware. If now in .init_instance() function, we should + * exit with error. If now at HNAE3_INIT_CLIENT stage of soft reset + * process, we should exit with error, and then HNAE3_INIT_CLIENT + * related process can rollback the operation like notifing hardware to + * free resources, HNAE3_INIT_CLIENT related process will exit with + * error to notify UBNIC driver to reschedule soft reset process once + * again. + */ + dev->dis_db = true; + + ret = read_poll_timeout_atomic(ops->ae_dev_reset_cnt, val, + val > dev->reset_cnt, HW_RESET_DELAY_US, + HW_RESET_TIMEOUT_US, false, handle); + if (!ret) + dev->is_reset = true; + + if (!dev->is_reset || reset_stage == UDMA_STATE_RST_INIT || + instance_stage == UDMA_STATE_INIT) + return CMD_RST_PRC_EBUSY; + + return CMD_RST_PRC_SUCCESS; +} + +static uint32_t udma_cmd_sw_resetting(struct udma_dev *dev) +{ + struct udma_priv *priv = (struct udma_priv *)dev->priv; + struct hnae3_handle *handle = priv->handle; + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + + /* When software reset is detected at .init_instance() function, we + * should stop sending mailbox&cmq&doorbell to hardware, and exit + * with error. + */ + dev->dis_db = true; + if (ops->ae_dev_reset_cnt(handle) != dev->reset_cnt) + dev->is_reset = true; + + return CMD_RST_PRC_EBUSY; +} + +static uint32_t check_aedev_reset_status(struct udma_dev *dev, + struct hnae3_handle *handle) +{ + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + uint64_t instance_stage; + uint64_t reset_stage; + uint64_t reset_cnt; + bool sw_resetting; + bool hw_resetting; + + /* The meaning of the following variables: + * reset_cnt -- The count value of completed hardware reset. + * hw_resetting -- Whether hardware device is resetting now. + * sw_resetting -- Whether UBNIC's software reset process is + * running now. + */ + instance_stage = handle->udmainfo.instance_state; + reset_stage = handle->udmainfo.reset_state; + reset_cnt = ops->ae_dev_reset_cnt(handle); + if (reset_cnt != dev->reset_cnt) + return udma_cmd_hw_reseted(dev, instance_stage, reset_stage); + + hw_resetting = ops->get_cmdq_stat(handle); + if (hw_resetting) + return udma_cmd_hw_resetting(dev, instance_stage, reset_stage); + + sw_resetting = ops->ae_dev_resetting(handle); + if (sw_resetting && instance_stage == UDMA_STATE_INIT) + return udma_cmd_sw_resetting(dev); + + return CMD_RST_PRC_OTHERS; +} + +bool udma_chk_mbox_is_avail(struct udma_dev *dev, bool *busy) +{ + struct udma_priv *priv = (struct udma_priv *)dev->priv; + uint32_t status; + + /* + * CMDQ fatal err or system reset means hw/fw is in an abnormal or + * unresponsive state, CMDQ command needs to return quickly, + * in order to avoid excessive printing, return CMD_RST_PRC_SUCCESS. + */ + if (dev->cmd.state == UDMA_CMDQ_STATE_FATAL_ERR || + dev->is_reset) + status = CMD_RST_PRC_SUCCESS; + else + status = check_aedev_reset_status(dev, priv->handle); + + *busy = (status == CMD_RST_PRC_EBUSY); + + return status == CMD_RST_PRC_OTHERS; +} + +static int udma_cmq_csq_done(struct udma_dev *dev) +{ + struct udma_priv *priv = (struct udma_priv *)dev->priv; + uint32_t tail = ub_read(dev, UDMA_TX_CMQ_CI_REG); + + return tail == priv->cmq.csq.head; +} + +void udma_cmq_setup_basic_desc(struct udma_cmq_desc *desc, + enum udma_opcode_type opcode, + bool is_read) +{ + memset((void *)desc, 0, sizeof(struct udma_cmq_desc)); + desc->opcode = cpu_to_le16(opcode); + desc->flag = + cpu_to_le16(UDMA_CMD_FLAG_NO_INTR | UDMA_CMD_FLAG_IN); + if (is_read) + desc->flag |= cpu_to_le16(UDMA_CMD_FLAG_WR); + else + desc->flag &= cpu_to_le16(~UDMA_CMD_FLAG_WR); +} + +void dump_desc(struct udma_dev *dev, + struct udma_cmq_desc *desc) +{ + if (desc->opcode == UDMA_OPC_QUERY_MB_ST) + return; + + if (((desc->data[SUB_OPCODE_IDX] & 0xFF) == + UDMA_CMD_WRITE_QPC_TIMER_BT0) || + ((desc->data[SUB_OPCODE_IDX] & 0xFF) == + UDMA_CMD_WRITE_CQC_TIMER_BT0)) + dev_err_ratelimited(dev->dev, + "Send cmd opcode:0x%4x, data: %08x %08x %08x %08x %08x %08x\n", + desc->opcode, desc->data[0], + desc->data[1], desc->data[2], + desc->data[3], desc->data[4], desc->data[5]); + else + dev_info(dev->dev, + "Send cmd opcode:0x%4x, data: %08x %08x %08x %08x %08x %08x\n", + desc->opcode, desc->data[0], + desc->data[1], desc->data[2], + desc->data[3], desc->data[4], desc->data[5]); +} + +static int __udma_cmq_send(struct udma_dev *dev, struct udma_cmq_desc *desc, + int num) +{ + struct udma_priv *priv = (struct udma_priv *)dev->priv; + struct udma_cmq_ring *csq = &priv->cmq.csq; + uint32_t timeout = 0; + uint16_t desc_ret; + uint32_t tail; + int ret = 0; + int i; + + tail = csq->head; + + mutex_lock(&csq->lock); + + for (i = 0; i < num; i++) { + csq->desc[csq->head++] = desc[i]; + if (csq->head == csq->desc_num) + csq->head = 0; + } + + /* barrier */ + wmb(); + + /* Write to hardware */ + ub_write(dev, UDMA_TX_CMQ_PI_REG, csq->head); + + do { + if (udma_cmq_csq_done(dev)) + break; + udelay(1); + } while (++timeout < priv->cmq.tx_timeout); + + if (udma_cmq_csq_done(dev)) { + for (i = 0; i < num; i++) { + /* check the result of hardware write back */ + desc[i] = csq->desc[tail++]; + if (tail == csq->desc_num) + tail = 0; + + desc_ret = le16_to_cpu(desc[i].retval); + if (likely(desc_ret == CMD_EXEC_SUCCESS)) + continue; + + dev_err_ratelimited(dev->dev, + "CMDQ IO error, opcode = %x, return = %x\n", + desc->opcode, desc_ret); + ret = -EIO; + } + } else { + /* FW/HW reset or incorrect number of desc */ + tail = ub_read(dev, UDMA_TX_CMQ_CI_REG); + dev_warn(dev->dev, "CMDQ move tail from %d to %d\n", + csq->head, tail); + csq->head = tail; + + dev->cmd.state = UDMA_CMDQ_STATE_HEAD_TAIL_ERR; + + ret = -EAGAIN; + } + + mutex_unlock(&csq->lock); + + return ret; +} + +int udma_cmq_send(struct udma_dev *dev, struct udma_cmq_desc *desc, int num) +{ + bool busy; + int ret; + + if (!udma_chk_mbox_is_avail(dev, &busy)) + return busy ? -EBUSY : 0; + + ret = __udma_cmq_send(dev, desc, num); + if (ret) { + if (!udma_chk_mbox_is_avail(dev, &busy)) + return busy ? -EBUSY : 0; + } + + return ret; +} + +static int udma_wait_mbox_complete(struct udma_dev *dev, uint32_t timeout, + uint8_t *complete_status) +{ + struct udma_mbox_status *mb_st; + struct udma_cmq_desc desc; + unsigned long end; + int ret = -EBUSY; + uint32_t status; + bool busy; + + mb_st = (struct udma_mbox_status *)desc.data; + end = msecs_to_jiffies(timeout) + jiffies; + while (udma_chk_mbox_is_avail(dev, &busy)) { + status = 0; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_QUERY_MB_ST, + true); + ret = __udma_cmq_send(dev, &desc, 1); + if (!ret) { + status = le32_to_cpu(mb_st->mb_status_hw_run); + /* No pending message exists in UDMA mbox. */ + if (!(status & MB_ST_HW_RUN_M)) + break; + } else if (!udma_chk_mbox_is_avail(dev, &busy)) { + break; + } + + if (time_after(jiffies, end)) { + dev_err_ratelimited(dev->dev, + "failed to wait mbox status 0x%x\n", + status); + return -ETIMEDOUT; + } + + cond_resched(); + ret = -EBUSY; + } + + if (!ret) { + *complete_status = (uint8_t)(status & MB_ST_COMPLETE_M); + } else if (!udma_chk_mbox_is_avail(dev, &busy)) { + /* Ignore all errors if the mbox is unavailable. */ + ret = busy ? -EBUSY : 0; + *complete_status = MB_ST_COMPLETE_M; + } + + return ret; +} + +static int __udma_post_mbox(struct udma_dev *dev, struct udma_cmq_desc *desc, + uint16_t token, int vfid_event) +{ + struct udma_mbox *mb = (struct udma_mbox *)desc->data; + + mb->token_event_en = cpu_to_le32(vfid_event + << UDMA_MB_EVENT_EN_SHIFT | token); + + return udma_cmq_send(dev, desc, 1); +} + +int udma_post_mbox(struct udma_dev *dev, struct udma_cmq_desc *desc, + uint16_t token, int vfid_event) +{ + uint8_t status = 0; + int ret; + + /* Waiting for the mbox to be idle */ + ret = udma_wait_mbox_complete(dev, UDMA_GO_BIT_TIMEOUT_MSECS, + &status); + if (unlikely(ret)) { + dev_err_ratelimited(dev->dev, + "failed to check post mbox status = 0x%x, ret = %d.\n", + status, ret); + return ret; + } + + /* Post new message to mbox */ + ret = __udma_post_mbox(dev, desc, token, vfid_event); + if (ret) + dev_err_ratelimited(dev->dev, + "failed to post mailbox, ret = %d.\n", ret); + + return ret; +} + +int udma_poll_mbox_done(struct udma_dev *dev, uint32_t timeout) +{ + uint8_t status = 0; + int ret; + + ret = udma_wait_mbox_complete(dev, timeout, &status); + if (!ret) { + if (status != MB_ST_COMPLETE_SUCC) + return -EBUSY; + } else { + dev_err_ratelimited(dev->dev, + "failed to check mbox status = 0x%x, ret = %d.\n", + status, ret); + } + + return ret; +} + +static int udma_cmd_mbox_post_hw(struct udma_dev *dev, + struct udma_cmq_desc *desc, + uint16_t token, int vfid_event) +{ + return dev->hw->post_mbox(dev, desc, token, vfid_event); +} + +static int __udma_cmd_mbox_poll(struct udma_dev *dev, + struct udma_cmq_desc *desc, + uint32_t timeout, int vfid) +{ + int vfid_event = (vfid << 1); + int ret, op; + + op = le32_to_cpu(((struct udma_mbox *)desc->data)->cmd_tag) & 0xff; + ret = udma_cmd_mbox_post_hw(dev, desc, CMD_POLL_TOKEN, vfid_event); + if (ret) { + dev_err_ratelimited(dev->dev, + "failed to post mailbox %x in poll mode, ret = %d.\n", + op, ret); + return ret; + } + + return dev->hw->poll_mbox_done(dev, timeout); +} + +static int udma_cmd_mbox_poll(struct udma_dev *dev, struct udma_cmq_desc *desc, + uint32_t timeout, int vfid) +{ + int ret; + + down(&dev->cmd.poll_sem); + ret = __udma_cmd_mbox_poll(dev, desc, timeout, vfid); + up(&dev->cmd.poll_sem); + + return ret; +} + +void udma_cmd_event(struct udma_dev *udma_dev, uint16_t token, uint8_t status, + uint64_t out_param) +{ + struct udma_cmd_context *ctx = + &udma_dev->cmd.context[token % udma_dev->cmd.max_cmds]; + + if (unlikely(token != ctx->token)) { + dev_err_ratelimited(udma_dev->dev, + "[cmd] invalid ae token 0x%x, context token is 0x%x.\n", + token, ctx->token); + return; + } + + ctx->result = (status == CMD_RST_PRC_SUCCESS) ? 0 : (-EIO); + ctx->out_param = out_param; + complete(&ctx->done); +} + +static int __udma_cmd_mbox_wait(struct udma_dev *udma_dev, + struct udma_cmq_desc *desc, + uint32_t timeout, int vfid) +{ + struct udma_cmdq *cmd = &udma_dev->cmd; + int vfid_event = (vfid << 1) | 0x1; + struct device *dev = udma_dev->dev; + struct udma_cmd_context *context; + int ret, op; + + spin_lock(&cmd->ctx_lock); + do { + context = &cmd->context[cmd->free_head]; + cmd->free_head = context->next; + } while (context->busy); + context->token += cmd->max_cmds; + context->busy = 1; + spin_unlock(&cmd->ctx_lock); + + reinit_completion(&context->done); + + op = le32_to_cpu(((struct udma_mbox *)desc->data)->cmd_tag) & 0xff; + ret = udma_cmd_mbox_post_hw(udma_dev, desc, context->token, vfid_event); + if (ret) { + dev_err_ratelimited(dev, + "failed to post mailbox %x in event mode, ret = %d.\n", + op, ret); + goto out; + } + + if (!wait_for_completion_timeout(&context->done, + msecs_to_jiffies(timeout))) { + dev_err_ratelimited(dev, "[cmd]token 0x%x of mailbox 0x%x timeout.\n", + context->token, op); + ret = -EBUSY; + goto out; + } + + ret = context->result; + if (ret) + dev_err_ratelimited(dev, "[cmd]token 0x%x of mailbox 0x%x error %d\n", + context->token, op, ret); + +out: + context->busy = 0; + return ret; +} + +static int udma_cmd_mbox_wait(struct udma_dev *dev, struct udma_cmq_desc *desc, + uint32_t timeout, int vfid) +{ + int ret; + + down(&dev->cmd.event_sem); + ret = __udma_cmd_mbox_wait(dev, desc, timeout, vfid); + up(&dev->cmd.event_sem); + + return ret; +} + +int udma_cmd_mbox(struct udma_dev *dev, struct udma_cmq_desc *desc, + uint32_t timeout, int vfid) +{ + bool is_busy; + + if (dev->hw->chk_mbox_avail) + if (!dev->hw->chk_mbox_avail(dev, &is_busy)) + return is_busy ? -EBUSY : 0; + + if (dev->cmd.use_events) + return udma_cmd_mbox_wait(dev, desc, timeout, vfid); + else + return udma_cmd_mbox_poll(dev, desc, timeout, vfid); +} diff --git a/drivers/ub/hw/hns3/hns3_udma_cmd.h b/drivers/ub/hw/hns3/hns3_udma_cmd.h new file mode 100644 index 0000000000000000000000000000000000000000..ece4ed394456f1d5cd07b4e216bc11e44a77fcfa --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_cmd.h @@ -0,0 +1,168 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_CMD_H +#define _UDMA_CMD_H + +#include "hns3_udma_device.h" + +#define CMD_MAX_NUM 32 +#define UDMA_MAILBOX_SIZE 4096 +#define UDMA_CMD_TIMEOUT_MSECS 10000 +#define CMD_POLL_TOKEN 0xffff +#define SUB_OPCODE_IDX 4 + +enum { + /* QPC BT commands */ + UDMA_CMD_WRITE_QPC_BT0 = 0x0, + UDMA_CMD_WRITE_QPC_BT1 = 0x1, + UDMA_CMD_WRITE_QPC_BT2 = 0x2, + UDMA_CMD_READ_QPC_BT0 = 0x4, + UDMA_CMD_READ_QPC_BT1 = 0x5, + UDMA_CMD_READ_QPC_BT2 = 0x6, + UDMA_CMD_DESTROY_QPC_BT0 = 0x8, + UDMA_CMD_DESTROY_QPC_BT1 = 0x9, + UDMA_CMD_DESTROY_QPC_BT2 = 0xa, + + /* QPC operation */ + UDMA_CMD_MODIFY_QPC = 0x41, + UDMA_CMD_QUERY_QPC = 0x42, + + UDMA_CMD_MODIFY_CQC = 0x52, + UDMA_CMD_QUERY_CQC = 0x53, + /* CQC BT commands */ + UDMA_CMD_WRITE_CQC_BT0 = 0x10, + UDMA_CMD_WRITE_CQC_BT1 = 0x11, + UDMA_CMD_WRITE_CQC_BT2 = 0x12, + UDMA_CMD_READ_CQC_BT0 = 0x14, + UDMA_CMD_READ_CQC_BT1 = 0x15, + UDMA_CMD_READ_CQC_BT2 = 0x1b, + UDMA_CMD_DESTROY_CQC_BT0 = 0x18, + UDMA_CMD_DESTROY_CQC_BT1 = 0x19, + UDMA_CMD_DESTROY_CQC_BT2 = 0x1a, + + /* MPT BT commands */ + UDMA_CMD_WRITE_MPT_BT0 = 0x20, + UDMA_CMD_WRITE_MPT_BT1 = 0x21, + UDMA_CMD_WRITE_MPT_BT2 = 0x22, + UDMA_CMD_READ_MPT_BT0 = 0x24, + UDMA_CMD_READ_MPT_BT1 = 0x25, + UDMA_CMD_READ_MPT_BT2 = 0x26, + UDMA_CMD_DESTROY_MPT_BT0 = 0x28, + UDMA_CMD_DESTROY_MPT_BT1 = 0x29, + UDMA_CMD_DESTROY_MPT_BT2 = 0x2a, + + /* CQC TIMER commands */ + UDMA_CMD_WRITE_CQC_TIMER_BT0 = 0x23, + UDMA_CMD_READ_CQC_TIMER_BT0 = 0x27, + + /* MPT commands */ + UDMA_CMD_CREATE_MPT = 0xd, + UDMA_CMD_DESTROY_MPT = 0xf, + UDMA_CMD_QUERY_MPT = 0x62, + + /* SRQC BT commands */ + UDMA_CMD_WRITE_SRQC_BT0 = 0x30, + UDMA_CMD_WRITE_SRQC_BT1 = 0x31, + UDMA_CMD_WRITE_SRQC_BT2 = 0x32, + UDMA_CMD_READ_SRQC_BT0 = 0x34, + UDMA_CMD_READ_SRQC_BT1 = 0x35, + UDMA_CMD_READ_SRQC_BT2 = 0x36, + UDMA_CMD_DESTROY_SRQC_BT0 = 0x38, + UDMA_CMD_DESTROY_SRQC_BT1 = 0x39, + UDMA_CMD_DESTROY_SRQC_BT2 = 0x3a, + + /* QPC TIMER commands */ + UDMA_CMD_WRITE_QPC_TIMER_BT0 = 0x33, + UDMA_CMD_READ_QPC_TIMER_BT0 = 0x37, + + /* QP/EE commands */ + UDMA_CMD_CREATE_SRQ = 0x70, + UDMA_CMD_MODIFY_SRQC = 0x72, + UDMA_CMD_QUERY_SRQC = 0x73, + UDMA_CMD_DESTROY_SRQ = 0x74, + + /* EQC commands */ + UDMA_CMD_CREATE_AEQC = 0x80, + UDMA_CMD_DESTROY_AEQC = 0x83, + UDMA_CMD_CREATE_CEQC = 0x90, + UDMA_CMD_DESTROY_CEQC = 0x93, + + /* SCC CTX BT commands */ + UDMA_CMD_READ_SCCC_BT0 = 0xa4, + UDMA_CMD_WRITE_SCCC_BT0 = 0xa5, + + /* CQ commands */ + UDMA_CMD_CREATE_CQC = 0x16, + UDMA_CMD_DESTROY_CQC = 0x17, + UDMA_CMD_RESERVED = 0xff, +}; + +enum { + CMD_RST_PRC_OTHERS, + CMD_RST_PRC_SUCCESS, + CMD_RST_PRC_EBUSY, +}; + +struct udma_mbox { + uint32_t in_param_l; + uint32_t in_param_h; + uint32_t out_param_l; + uint32_t out_param_h; + uint32_t cmd_tag; + uint32_t token_event_en; +}; + +struct udma_mbox_status { + uint32_t mb_status_hw_run; + uint32_t rsv[5]; +}; + +#define UDMA_GO_BIT_TIMEOUT_MSECS 10000 + +#define MB_ST_HW_RUN_M BIT(31) +#define MB_ST_COMPLETE_M GENMASK(7, 0) + +#define MB_ST_COMPLETE_SUCC 1 +#define UDMA_MB_EVENT_EN_SHIFT 16 + +void dump_desc(struct udma_dev *dev, struct udma_cmq_desc *desc); +struct udma_cmd_mailbox *udma_alloc_cmd_mailbox(struct udma_dev *dev); +void udma_free_cmd_mailbox(struct udma_dev *dev, + struct udma_cmd_mailbox *mailbox); +int udma_post_mbox(struct udma_dev *dev, struct udma_cmq_desc *desc, + uint16_t token, int vfid_event); +int udma_poll_mbox_done(struct udma_dev *dev, uint32_t timeout); +bool udma_chk_mbox_is_avail(struct udma_dev *dev, bool *busy); +void udma_cmq_setup_basic_desc(struct udma_cmq_desc *desc, + enum udma_opcode_type opcode, + bool is_read); +int udma_cmq_send(struct udma_dev *dev, struct udma_cmq_desc *desc, int num); +int udma_cmd_mbox(struct udma_dev *dev, struct udma_cmq_desc *desc, + uint32_t timeout, int vfid); +void udma_cmd_event(struct udma_dev *udma_dev, uint16_t token, uint8_t status, + uint64_t out_param); +static inline void mbox_desc_init(struct udma_mbox *mb, uint64_t in_param, + uint64_t out_param, uint32_t in_modifier, + uint16_t op) +{ + mb->in_param_l = cpu_to_le32(in_param); + mb->in_param_h = cpu_to_le32(in_param >> 32); + mb->out_param_l = cpu_to_le32(out_param); + mb->out_param_h = cpu_to_le32(out_param >> 32); + mb->cmd_tag = cpu_to_le32(in_modifier << 8 | op); +} + +#endif /* _UDMA_CMD_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_common.h b/drivers/ub/hw/hns3/hns3_udma_common.h new file mode 100644 index 0000000000000000000000000000000000000000..588eb1fa091b22708574dac231e9333921b24a5e --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_common.h @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_COMMON_H +#define _UDMA_COMMON_H +#include + +#define ub_write(dev, reg, val) writel((val), (dev)->reg_base + (reg)) +#define ub_read(dev, reg) readl((dev)->reg_base + (reg)) + +#define UDMA_UDP_DPORT 4791 +#define DMA_WQE_SHIFT 3 +#define DMA_DB_RECORD_SHIFT 1 + +#define udma_get_field(origin, mask, shift) \ + ((le32_to_cpu(origin) & (mask)) >> (uint32_t)(shift)) +#define udma_get_field64(origin, mask, shift) \ + ((le64_to_cpu(origin) & (mask)) >> (uint32_t)(shift)) +#define udma_get_bit(origin, shift) \ + udma_get_field((origin), (1ul << (shift)), (shift)) + +#define udma_set_field(origin, mask, shift, val) \ + do { \ + (origin) &= ~cpu_to_le32(mask); \ + (origin) |= cpu_to_le32(((uint32_t)(val) << \ + (uint32_t)(shift)) & (mask)); \ + } while (0) + +#define udma_set_bit(origin, shift, val) \ + udma_set_field((origin), (1ul << (shift)), (shift), (val)) + +#define _udma_reg_enable(ptr, field) \ + ({ \ + const uint32_t *_ptr = (uint32_t *)(ptr); \ + *((uint32_t *)_ptr + ((field) >> 32) / 32) |= cpu_to_le32( \ + BIT((((field) << 32) >> 32) % 32)); \ + }) + +#define udma_reg_enable(ptr, field) _udma_reg_enable(ptr, field) + +#define _udma_reg_clear(ptr, field) \ + ({ \ + const uint32_t *_ptr = (uint32_t *)(ptr); \ + BUILD_BUG_ON((((field) >> 32) / 32) != ((((field) << 32) >> 32) / 32)); \ + *((uint32_t *)_ptr + ((field) >> 32) / 32) &= \ + ~cpu_to_le32(GENMASK(((field) >> 32) % 32, (((field) << 32) >> 32) % 32)); \ + }) + +#define udma_reg_clear(ptr, field) _udma_reg_clear(ptr, field) + +#define _udma_reg_write(ptr, field, val) \ + ({ \ + uint32_t _val = val; \ + _udma_reg_clear((ptr), field); \ + *((uint32_t *)(ptr) + ((field) >> 32) / 32) |= \ + cpu_to_le32(FIELD_PREP(GENMASK(((field) >> 32) % 32, \ + (((field) << 32) >> 32) % 32), _val)); \ + }) + +#define udma_reg_write(ptr, field, val) _udma_reg_write(ptr, field, val) + +#define _udma_reg_read(ptr, field) \ + ({ \ + const uint32_t *_ptr = (uint32_t *)(ptr); \ + BUILD_BUG_ON((((field) >> 32) / 32) != ((((field) << 32) >> 32) / 32)); \ + FIELD_GET(GENMASK(((field) >> 32) % 32, (((field) << 32) >> 32) % 32), \ + le32_to_cpu(*((uint32_t *)_ptr + ((field) >> 32) / 32))); \ + }) + +#define udma_reg_read(ptr, field) _udma_reg_read(ptr, field) + +#endif /* _UDMA_COMMON_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_db.c b/drivers/ub/hw/hns3/hns3_udma_db.c new file mode 100644 index 0000000000000000000000000000000000000000..0880d4d4bcfdc791f14c43623abaf847f5f30ab8 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_db.c @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include +#include +#include "urma/ubcore_api.h" +#include "hns3_udma_device.h" +#include "hns3_udma_db.h" + +int udma_db_map_user(struct udma_dev *udma_dev, uint64_t virt, + struct udma_db *db) +{ + uint64_t page_addr = virt & PAGE_MASK; + union ubcore_umem_flag access = {}; + struct udma_user_db_page *db_page; + uint32_t offset; + int ret = 0; + + mutex_lock(&udma_dev->pgdir_mutex); + + list_for_each_entry(db_page, &udma_dev->pgdir_list, list) { + if (db_page->user_virt == page_addr) + goto found; + } + + db_page = kmalloc(sizeof(*db_page), GFP_KERNEL); + if (!db_page) { + ret = -ENOMEM; + goto out; + } + + refcount_set(&db_page->refcount, 1); + db_page->user_virt = page_addr; + access.bs.non_pin = 0; + access.bs.writable = 1; + db_page->umem = ubcore_umem_get(&udma_dev->ub_dev, page_addr, + PAGE_SIZE, access); + if (IS_ERR(db_page->umem)) { + ret = PTR_ERR(db_page->umem); + kfree(db_page); + goto out; + } + + list_add(&db_page->list, &udma_dev->pgdir_list); + +found: + offset = virt - page_addr; + db->dma = sg_dma_address(db_page->umem->sg_head.sgl) + offset; + db->virt_addr = (char *)sg_virt(db_page->umem->sg_head.sgl) + offset; + db->user_page = db_page; + refcount_inc(&db_page->refcount); + +out: + mutex_unlock(&udma_dev->pgdir_mutex); + + return ret; +} + +void udma_db_unmap_user(struct udma_dev *udma_dev, struct udma_db *db) +{ + mutex_lock(&udma_dev->pgdir_mutex); + + refcount_dec(&db->user_page->refcount); + if (refcount_dec_if_one(&db->user_page->refcount)) { + list_del(&db->user_page->list); + ubcore_umem_release(db->user_page->umem); + kfree(db->user_page); + } + + mutex_unlock(&udma_dev->pgdir_mutex); +} diff --git a/drivers/ub/hw/hns3/hns3_udma_db.h b/drivers/ub/hw/hns3/hns3_udma_db.h new file mode 100644 index 0000000000000000000000000000000000000000..e5a230cf49c0048a5410fc36012581a35aee217e --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_db.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_DB_H +#define _UDMA_DB_H + +#include "hns3_udma_device.h" + +int udma_db_map_user(struct udma_dev *udma_dev, uint64_t virt, + struct udma_db *db); + +void udma_db_unmap_user(struct udma_dev *udma_dev, struct udma_db *db); + +#endif /* _UDMA_DB_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_dca.c b/drivers/ub/hw/hns3/hns3_udma_dca.c new file mode 100644 index 0000000000000000000000000000000000000000..be71d355792cbe2c6bd96c0d8512cfe853ed103b --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_dca.c @@ -0,0 +1,1235 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include +#include "urma/ubcore_api.h" +#include "hns3_udma_cmd.h" +#include "hns3_udma_dca.h" + +static void travel_dca_pages(struct udma_dca_ctx *ctx, void *param, + int (*cb)(struct dca_mem *, uint32_t, void *)) +{ + struct dca_mem *mem; + unsigned long flags; + uint32_t i; + bool avail; + int ret; + + spin_lock_irqsave(&ctx->pool_lock, flags); + list_for_each_entry(mem, &ctx->pool, list) { + spin_unlock_irqrestore(&ctx->pool_lock, flags); + + spin_lock(&mem->lock); + avail = dca_mem_is_available(mem); + ret = 0; + for (i = 0; avail && i < mem->page_count; i++) { + ret = cb(mem, i, param); + if (ret == DCA_MEM_STOP_ITERATE || + ret == DCA_MEM_NEXT_ITERATE) + break; + } + spin_unlock(&mem->lock); + spin_lock_irqsave(&ctx->pool_lock, flags); + + if (ret == DCA_MEM_STOP_ITERATE) + goto done; + } + +done: + spin_unlock_irqrestore(&ctx->pool_lock, flags); +} + +void udma_enable_dca(struct udma_dev *dev, struct udma_qp *qp) +{ + struct udma_dca_cfg *cfg = &qp->dca_cfg; + + spin_lock_init(&cfg->lock); + INIT_LIST_HEAD(&cfg->aging_node); + cfg->buf_id = UDMA_DCA_INVALID_BUF_ID; + cfg->npages = qp->buff_size >> UDMA_HW_PAGE_SHIFT; + cfg->dcan = UDMA_DCA_INVALID_DCA_NUM; +} + +static void stop_aging_dca_mem(struct udma_dca_ctx *ctx, + struct udma_dca_cfg *cfg, bool stop_worker) +{ + spin_lock(&ctx->aging_lock); + if (stop_worker) { + ctx->exit_aging = true; + cancel_delayed_work(&ctx->aging_dwork); + } + + spin_lock(&cfg->lock); + + if (!list_empty(&cfg->aging_node)) + list_del_init(&cfg->aging_node); + + spin_unlock(&cfg->lock); + spin_unlock(&ctx->aging_lock); +} + +static void update_dca_buf_status(struct udma_dca_ctx *ctx, uint32_t dcan, + bool en) +{ + uintptr_t *st = ctx->buf_status; + + if (st && dcan < ctx->max_qps) { + if (en) + set_bit(DCAN_TO_STAT_BIT(dcan), st); + else + clear_bit(DCAN_TO_STAT_BIT(dcan), st); + /* barrier */ + smp_mb__after_atomic(); + } +} + +static int free_buffer_pages_proc(struct dca_mem *mem, uint32_t index, + void *param) +{ + struct dca_page_free_buf_attr *attr = param; + struct dca_page_state *state; + uint32_t free_pages = 0; + bool changed = false; + bool stop = false; + uint32_t i = 0; + + for (; !stop && i < mem->page_count; i++) { + state = &mem->states[i]; + /* Change matched pages state */ + if (dca_page_is_attached(state, attr->buf_id)) { + set_dca_page_to_free(state); + changed = true; + attr->free_pages++; + if (attr->free_pages == attr->max_pages) + stop = true; + } + + if (dca_page_is_free(state)) + free_pages++; + } + + for (; changed && i < mem->page_count; i++) { + state = &mem->states[i]; + if (dca_page_is_free(state)) + free_pages++; + } + + if (changed && free_pages == mem->page_count) + attr->clean_mems++; + + return stop ? DCA_MEM_STOP_ITERATE : DCA_MEM_NEXT_ITERATE; +} + +static void free_buf_from_dca_mem(struct udma_dca_ctx *ctx, + struct udma_dca_cfg *cfg) +{ + struct dca_page_free_buf_attr attr = {}; + unsigned long flags; + uint32_t buf_id; + + update_dca_buf_status(ctx, cfg->dcan, false); + spin_lock(&cfg->lock); + buf_id = cfg->buf_id; + cfg->buf_id = UDMA_DCA_INVALID_BUF_ID; + spin_unlock(&cfg->lock); + if (buf_id == UDMA_DCA_INVALID_BUF_ID) + return; + + attr.buf_id = buf_id; + attr.max_pages = cfg->npages; + travel_dca_pages(ctx, &attr, free_buffer_pages_proc); + + /* Update free size */ + spin_lock_irqsave(&ctx->pool_lock, flags); + ctx->free_mems += attr.clean_mems; + ctx->free_size += attr.free_pages << UDMA_HW_PAGE_SHIFT; + spin_unlock_irqrestore(&ctx->pool_lock, flags); +} + +static void restart_aging_dca_mem(struct udma_dev *dev, + struct udma_dca_ctx *ctx) +{ + spin_lock(&ctx->aging_lock); + ctx->exit_aging = false; + if (!list_empty(&ctx->aging_new_list)) + queue_delayed_work(dev->irq_workq, &ctx->aging_dwork, + msecs_to_jiffies(DCA_MEM_AGEING_MSES)); + + spin_unlock(&ctx->aging_lock); +} + +static void kick_dca_buf(struct udma_dev *dev, struct udma_dca_cfg *cfg, + struct udma_dca_ctx *ctx) +{ + stop_aging_dca_mem(ctx, cfg, true); + free_buf_from_dca_mem(ctx, cfg); + restart_aging_dca_mem(dev, ctx); +} + +static void free_dca_num(struct udma_dca_cfg *cfg, struct udma_dca_ctx *ctx) +{ + if (cfg->dcan == UDMA_DCA_INVALID_DCA_NUM) + return; + + ida_free(&ctx->ida, cfg->dcan); + cfg->dcan = UDMA_DCA_INVALID_DCA_NUM; +} + +void udma_disable_dca(struct udma_dev *dev, struct udma_qp *qp) +{ + struct udma_dca_cfg *cfg = &qp->dca_cfg; + struct udma_dca_ctx *ctx = qp->dca_ctx; + + kick_dca_buf(dev, cfg, ctx); + free_dca_num(cfg, ctx); +} + +static struct dca_mem *alloc_dca_mem(struct udma_dev *dev, + struct udma_dca_ctx *ctx) +{ + struct dca_mem *mem, *found = NULL; + unsigned long flags; + + spin_lock_irqsave(&ctx->pool_lock, flags); + list_for_each_entry(mem, &ctx->pool, list) { + spin_lock(&mem->lock); + if (!mem->flags) { + found = mem; + mem->flags |= DCA_MEM_FLAGS_ALLOCED; + spin_unlock(&mem->lock); + break; + } + spin_unlock(&mem->lock); + } + spin_unlock_irqrestore(&ctx->pool_lock, flags); + + if (found) + return found; + + mem = kzalloc(sizeof(*mem), GFP_ATOMIC); + if (!mem) + return NULL; + + spin_lock_init(&mem->lock); + INIT_LIST_HEAD(&mem->list); + + mem->flags |= DCA_MEM_FLAGS_ALLOCED; + + spin_lock_irqsave(&ctx->pool_lock, flags); + list_add(&mem->list, &ctx->pool); + spin_unlock_irqrestore(&ctx->pool_lock, flags); + + return mem; +} + +static void free_dca_mem(struct dca_mem *mem) +{ + /* When iterate all DCA mems in travel_dca_pages(), we will NOT hold the + * pool's lock and just set the DCA mem as free state during the DCA is + * working until cleanup the DCA context in cleanup_dca_context(). + */ + spin_lock(&mem->lock); + mem->flags = 0; + spin_unlock(&mem->lock); +} + +static void *alloc_dca_pages(struct udma_dev *dev, struct dca_mem *mem, + struct udma_dca_reg_attr *attr) +{ + struct ubcore_device *ub_dev = &dev->ub_dev; + union ubcore_umem_flag flag = {}; + struct ubcore_umem *umem; + + flag.bs.non_pin = 0; + flag.bs.writable = 1; + umem = ubcore_umem_get(ub_dev, attr->addr, attr->size, flag); + if (IS_ERR_OR_NULL(umem)) { + dev_err(dev->dev, "failed to get uDCA pages, ret = %ld.\n", + PTR_ERR(umem)); + return NULL; + } + + mem->page_count = umem_cal_npages(umem->va, umem->length); + + return umem; +} + +static void init_dca_umem_states(struct udma_dev *dev, struct ubcore_umem *umem, + struct dca_page_state *states, uint32_t count) +{ + uint32_t npage_per_sg, k, i, total = 0; + dma_addr_t cur_addr, pre_addr = 0; + struct scatterlist *sg; + + for_each_sg(umem->sg_head.sgl, sg, umem->sg_head.nents, k) { + npage_per_sg = sg_dma_len(sg) >> UDMA_HW_PAGE_SHIFT; + for (i = 0; i < npage_per_sg; ++i) { + cur_addr = sg_dma_address(sg) + (i << UDMA_HW_PAGE_SHIFT); + if (cur_addr - pre_addr != UDMA_PAGE_SIZE) + states[total].head = 1; + + if (count <= ++total) + return; + + pre_addr = cur_addr; + } + } +} + +static struct dca_page_state *alloc_dca_states(struct udma_dev *dev, + void *pages, uint32_t count) +{ + struct dca_page_state *states; + + states = kcalloc(count, sizeof(*states), GFP_KERNEL); + if (!states) + return NULL; + + init_dca_umem_states(dev, pages, states, count); + + return states; +} + +static void stop_free_dca_buf(struct udma_dca_ctx *ctx, uint32_t dcan) +{ + uintptr_t *st = ctx->sync_status; + + if (st && dcan < ctx->max_qps) + clear_bit_unlock(DCAN_TO_SYNC_BIT(dcan), st); +} + +static uint32_t alloc_dca_num(struct udma_dca_ctx *ctx) +{ + int ret; + + ret = ida_alloc_range(&ctx->ida, 0, ctx->max_qps - 1, GFP_KERNEL); + if (ret < 0) + return UDMA_DCA_INVALID_DCA_NUM; + + stop_free_dca_buf(ctx, ret); + update_dca_buf_status(ctx, ret, false); + return ret; +} + +void udma_modify_dca(struct udma_dev *dev, struct udma_qp *qp) +{ + struct udma_dca_cfg *cfg = &qp->dca_cfg; + struct udma_dca_ctx *ctx = qp->dca_ctx; + + if (qp->state == QPS_RESET || qp->state == QPS_ERR) { + kick_dca_buf(dev, cfg, ctx); + free_dca_num(cfg, ctx); + } else if (qp->state == QPS_RTS) { + free_dca_num(cfg, ctx); + cfg->dcan = alloc_dca_num(ctx); + } +} + +int udma_register_dca_mem(struct udma_dev *dev, struct udma_ucontext *context, + struct udma_dca_reg_attr *attr) +{ + struct udma_dca_ctx *ctx = &context->dca_ctx; + struct dca_mem *mem; + unsigned long flags; + void *states; + void *pages; + + mem = alloc_dca_mem(dev, ctx); + if (!mem) { + dev_err(dev->dev, "failed to alloc dca mem.\n"); + return -ENOMEM; + } + + pages = alloc_dca_pages(dev, mem, attr); + if (!pages) { + dev_err(dev->dev, "failed to alloc dca pages.\n"); + goto err_alloc_dca_pages; + } + + states = alloc_dca_states(dev, pages, mem->page_count); + if (!states) { + dev_err(dev->dev, "failed to alloc dca states.\n"); + goto err_alloc_dca_states; + } + + spin_lock_irqsave(&ctx->pool_lock, flags); + + spin_lock(&mem->lock); + mem->pages = pages; + mem->states = states; + mem->key = attr->key; + mem->size = attr->size; + mem->flags |= DCA_MEM_FLAGS_REGISTERED; + spin_unlock(&mem->lock); + + ctx->free_mems++; + ctx->free_size += attr->size; + ctx->total_size += attr->size; + spin_unlock_irqrestore(&ctx->pool_lock, flags); + + return 0; +err_alloc_dca_states: + ubcore_umem_release(pages); +err_alloc_dca_pages: + free_dca_mem(mem); + + return -ENOMEM; +} + +void unregister_dca_mem(struct udma_dev *dev, struct udma_dca_ctx *ctx, + struct dca_mem *mem) +{ + void *states, *pages; + unsigned long flags; + + spin_lock_irqsave(&ctx->pool_lock, flags); + + spin_lock(&mem->lock); + mem->flags &= ~DCA_MEM_FLAGS_REGISTERED; + mem->page_count = 0; + pages = mem->pages; + mem->pages = NULL; + states = mem->states; + mem->states = NULL; + spin_unlock(&mem->lock); + + ctx->free_mems--; + ctx->free_size -= mem->size; + + ctx->total_size -= mem->size; + spin_unlock_irqrestore(&ctx->pool_lock, flags); + + kfree(states); + ubcore_umem_release(pages); +} + +static uint32_t get_udca_max_qps(struct udma_dev *udma_dev, + struct udma_create_ctx_ucmd *ucmd) +{ + uint32_t qp_num; + + if (ucmd->comp & UDMA_CONTEXT_MASK_DCA_PRIME_QPS) { + qp_num = ucmd->dca_max_qps; + if (!qp_num) + qp_num = udma_dev->caps.num_qps; + } else { + qp_num = 0; + } + + return qp_num; +} + +static bool start_free_dca_buf(struct udma_dca_ctx *ctx, uint32_t dcan) +{ + uintptr_t *st = ctx->sync_status; + + if (st && dcan < ctx->max_qps) + return !test_and_set_bit_lock(dcan, st); + + return true; +} + +static int udma_query_qpc(struct udma_dev *udma_dev, uint32_t qpn, + void *context) +{ + struct udma_cmd_mailbox *mailbox; + struct udma_cmq_desc desc; + struct udma_mbox *mb; + int ret; + + mailbox = udma_alloc_cmd_mailbox(udma_dev); + if (IS_ERR_OR_NULL(mailbox)) { + dev_err(udma_dev->dev, "alloc mailbox failed\n"); + ret = PTR_ERR(mailbox); + goto alloc_mailbox_fail; + } + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mb = (struct udma_mbox *)desc.data; + mbox_desc_init(mb, 0, mailbox->dma, qpn, UDMA_CMD_QUERY_QPC); + + ret = udma_cmd_mbox(udma_dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); + if (ret) { + dev_err(udma_dev->dev, "QUERY id(0x%x) cmd(0x%x) error(%d).\n", + qpn, UDMA_CMD_QUERY_QPC, ret); + goto err_mailbox; + } + memcpy(context, mailbox->buf, sizeof(struct udma_qp_context)); + +err_mailbox: + udma_free_cmd_mailbox(udma_dev, mailbox); +alloc_mailbox_fail: + return ret; +} + +static bool udma_chk_dca_buf_inactive(struct udma_dev *udma_dev, + struct udma_qp *qp) +{ + struct udma_dca_cfg *cfg = &qp->dca_cfg; + struct udma_qp_context context = {}; + uint32_t tmp, sq_idx; + int state; + int ret; + + ret = udma_query_qpc(udma_dev, qp->qpn, &context); + if (ret) { + dev_info(udma_dev->dev, "Failed to query QPC, ret = %d.\n", + ret); + return false; + } + + state = udma_reg_read(&context, QPC_QP_ST); + if (state == QP_ST_ERR || state == QP_ST_RST) + return true; + + if (qp->sq.wqe_cnt > 0) { + tmp = udma_reg_read(&context, QPC_RETRY_MSG_MSN); + sq_idx = tmp & (qp->sq.wqe_cnt - 1); + /* If SQ-PI equals to retry_msg_msn in QPC, the QP is + * inactive. + */ + if (sq_idx != cfg->sq_idx) + return false; + } + + return true; +} + +static void process_aging_dca_mem(struct udma_dev *udma_dev, + struct udma_dca_ctx *ctx) +{ + struct udma_dca_cfg *cfg, *cfg_tmp; + struct udma_qp *qp; + + spin_lock(&ctx->aging_lock); + list_for_each_entry_safe(cfg, cfg_tmp, &ctx->aging_new_list, aging_node) + list_move(&cfg->aging_node, &ctx->aging_proc_list); + + while (!ctx->exit_aging && !list_empty(&ctx->aging_proc_list)) { + cfg = list_first_entry(&ctx->aging_proc_list, + struct udma_dca_cfg, aging_node); + list_del_init_careful(&cfg->aging_node); + qp = container_of(cfg, struct udma_qp, dca_cfg); + spin_unlock(&ctx->aging_lock); + + if (start_free_dca_buf(ctx, cfg->dcan)) { + if (udma_chk_dca_buf_inactive(udma_dev, qp)) + free_buf_from_dca_mem(ctx, cfg); + + stop_free_dca_buf(ctx, cfg->dcan); + } + + spin_lock(&ctx->aging_lock); + + spin_lock(&cfg->lock); + + if (cfg->buf_id != UDMA_DCA_INVALID_BUF_ID) + list_move(&cfg->aging_node, &ctx->aging_new_list); + + spin_unlock(&cfg->lock); + } + spin_unlock(&ctx->aging_lock); +} + +static void udca_mem_aging_work(struct work_struct *work) +{ + struct udma_dca_ctx *ctx = container_of(work, struct udma_dca_ctx, + aging_dwork.work); + struct udma_ucontext *ucontext = container_of(ctx, struct udma_ucontext, + dca_ctx); + struct udma_dev *udma_dev = container_of(ucontext->uctx.ub_dev, + struct udma_dev, ub_dev); + + cancel_delayed_work(&ctx->aging_dwork); + process_aging_dca_mem(udma_dev, ctx); + if (!ctx->exit_aging) + restart_aging_dca_mem(udma_dev, ctx); +} + +static void init_dca_context(struct udma_dca_ctx *dca_ctx) +{ + INIT_LIST_HEAD(&dca_ctx->pool); + spin_lock_init(&dca_ctx->pool_lock); + dca_ctx->total_size = 0; + + ida_init(&dca_ctx->ida); + INIT_LIST_HEAD(&dca_ctx->aging_new_list); + INIT_LIST_HEAD(&dca_ctx->aging_proc_list); + spin_lock_init(&dca_ctx->aging_lock); + dca_ctx->exit_aging = false; + + INIT_DELAYED_WORK(&dca_ctx->aging_dwork, udca_mem_aging_work); +} + +static void init_udca_status(struct udma_ucontext *uctx, int udca_max_qps, + uint32_t dev_max_qps) +{ + const uint32_t bits_per_qp = 2 * UDMA_DCA_BITS_PER_STATUS; + struct udma_dca_ctx *dca_ctx = &uctx->dca_ctx; + void *kaddr; + size_t size; + + size = BITS_TO_BYTES(udca_max_qps * bits_per_qp); + dca_ctx->status_npage = DIV_ROUND_UP(size, PAGE_SIZE); + + size = dca_ctx->status_npage * PAGE_SIZE; + dca_ctx->max_qps = min_t(uint32_t, dev_max_qps, + size * BITS_PER_BYTE / bits_per_qp); + + kaddr = alloc_pages_exact(size, GFP_KERNEL | __GFP_ZERO); + if (!kaddr) + return; + + dca_ctx->buf_status = (uintptr_t *)kaddr; + dca_ctx->sync_status = (uintptr_t *)(kaddr + size / DCA_BITS_HALF); +} + +int udma_register_udca(struct udma_dev *udma_dev, + struct udma_ucontext *context, struct ubcore_udrv_priv *udrv_data) +{ + struct udma_dca_ctx *dca_ctx = &context->dca_ctx; + struct udma_create_ctx_ucmd ucmd = {}; + int max_qps; + int ret; + + ret = copy_from_user(&ucmd, (void *)udrv_data->in_addr, + min(udrv_data->in_len, (uint32_t)sizeof(ucmd))); + if (ret) { + dev_err(udma_dev->dev, "Failed to copy udata, ret = %d.\n", + ret); + return -EFAULT; + } + + if (!(udma_dev->caps.flags & UDMA_CAP_FLAG_DCA_MODE) || + ucmd.dca_unit_size == 0) + return 0; + + dca_ctx->unit_size = ucmd.dca_unit_size; + max_qps = get_udca_max_qps(udma_dev, &ucmd); + init_dca_context(dca_ctx); + if (max_qps > 0) + init_udca_status(context, max_qps, udma_dev->caps.num_qps); + + return 0; +} + +static void cleanup_dca_context(struct udma_dca_ctx *ctx) +{ + struct dca_mem *mem, *tmp; + unsigned long flags; + + spin_lock(&ctx->aging_lock); + cancel_delayed_work_sync(&ctx->aging_dwork); + spin_unlock(&ctx->aging_lock); + + spin_lock_irqsave(&ctx->pool_lock, flags); + list_for_each_entry_safe(mem, tmp, &ctx->pool, list) { + list_del(&mem->list); + spin_lock(&mem->lock); + mem->flags = 0; + spin_unlock(&mem->lock); + spin_unlock_irqrestore(&ctx->pool_lock, flags); + + kfree(mem->states); + ubcore_umem_release(mem->pages); + kfree(mem); + + spin_lock_irqsave(&ctx->pool_lock, flags); + } + ctx->total_size = 0; + spin_unlock_irqrestore(&ctx->pool_lock, flags); +} + +void udma_unregister_udca(struct udma_dev *udma_dev, + struct udma_ucontext *context) +{ + struct udma_dca_ctx *dca_ctx = &context->dca_ctx; + + if (dca_ctx->unit_size == 0) + return; + + cleanup_dca_context(dca_ctx); + if (dca_ctx->buf_status) { + free_pages_exact(dca_ctx->buf_status, + dca_ctx->status_npage * PAGE_SIZE); + dca_ctx->buf_status = NULL; + } + + ida_destroy(&dca_ctx->ida); +} + +static int dereg_dca_page_proc(struct dca_mem *mem, uint32_t index, + void *param) +{ + struct udma_dca_dereg_attr *attr = param; + + if (mem->key == attr->free_key) { + attr->mem = mem; + return DCA_MEM_STOP_ITERATE; + } else { + return DCA_MEM_NEXT_ITERATE; + } +} + +int udma_deregister_dca_mem(struct udma_dev *dev, + struct udma_ucontext *context, + struct udma_dca_dereg_attr *attr, bool from_user) +{ + if (from_user) { + travel_dca_pages(&context->dca_ctx, attr, + dereg_dca_page_proc); + if (attr->mem == NULL) { + dev_err(dev->dev, "failed to dereg DCA mems.\n"); + return -EINVAL; + } + } + + unregister_dca_mem(dev, &context->dca_ctx, attr->mem); + free_dca_mem(attr->mem); + + return 0; +} + +static int shrink_dca_page_proc(struct dca_mem *mem, uint32_t index, + void *param) +{ + struct udma_dca_shrink_resp *resp = param; + struct dca_page_state *state; + int i, free_pages; + + free_pages = 0; + for (i = 0; i < mem->page_count; i++) { + state = &mem->states[i]; + if (dca_page_is_free(state)) + free_pages++; + } + + /* No any page be used */ + if (free_pages == mem->page_count) { + /* shrink first empty DCA mem */ + if (!resp->free_mems) { + resp->mem = mem; + resp->free_key = mem->key; + } + resp->free_mems++; + } + + if (resp->free_mems > 1) + return DCA_MEM_STOP_ITERATE; + else + return DCA_MEM_NEXT_ITERATE; +} + +void udma_shrink_dca_mem(struct udma_dev *dev, struct udma_ucontext *context, + struct udma_dca_shrink_attr *attr, + struct udma_dca_shrink_resp *resp) +{ + struct udma_dca_ctx *ctx = &context->dca_ctx; + unsigned long flags; + bool need_shink; + + spin_lock_irqsave(&ctx->pool_lock, flags); + need_shink = ctx->free_mems > 0 && ctx->free_size > attr->reserved_size; + spin_unlock_irqrestore(&ctx->pool_lock, flags); + if (!need_shink) + return; + + travel_dca_pages(ctx, resp, shrink_dca_page_proc); +} + +static int query_dca_active_pages_proc(struct dca_mem *mem, uint32_t index, + void *param) +{ + struct dca_page_state *state = &mem->states[index]; + struct dca_page_query_active_attr *attr = param; + + if (!dca_page_is_active(state, attr->buf_id)) + return 0; + + if (attr->curr_index < attr->start_index) { + attr->curr_index++; + return 0; + } else if (attr->curr_index > attr->start_index) { + return DCA_MEM_STOP_ITERATE; + } + + /* Search first page in DCA mem */ + attr->page_index = index; + attr->mem_key = mem->key; + /* Search active pages in continuous addresses */ + while (index < mem->page_count) { + state = &mem->states[index]; + if (!dca_page_is_active(state, attr->buf_id)) + break; + + index++; + attr->page_count++; + } + + return DCA_MEM_STOP_ITERATE; +} + +int udma_query_dca_mem(struct udma_dev *dev, struct udma_dca_query_attr *attr, + struct udma_dca_query_resp *resp) +{ + struct dca_page_query_active_attr a_attr = {}; + struct udma_dca_ctx *ctx; + struct udma_dca_cfg *cfg; + struct udma_qp *qp; + + qp = get_qp(dev, attr->qpn); + if (qp == NULL) { + dev_err(dev->dev, "failed to find qp, qpn = 0x%llx\n", attr->qpn); + return -EINVAL; + } + cfg = &qp->dca_cfg; + ctx = qp->dca_ctx; + + a_attr.buf_id = qp->dca_cfg.buf_id; + a_attr.start_index = attr->page_idx; + travel_dca_pages(ctx, &a_attr, query_dca_active_pages_proc); + + resp->mem_key = a_attr.mem_key; + resp->mem_ofs = a_attr.page_index << UDMA_HW_PAGE_SHIFT; + resp->page_count = a_attr.page_count; + + return a_attr.page_count ? 0 : -ENOMEM; +} + +static bool dca_page_is_allocable(struct dca_page_state *state, bool head) +{ + bool is_free = dca_page_is_free(state) || dca_page_is_inactive(state); + + return head ? is_free : is_free && !state->head; +} + +static int assign_dca_pages_proc(struct dca_mem *mem, uint32_t index, + void *param) +{ + struct dca_page_assign_attr *attr = param; + struct dca_page_state *state; + uint32_t checked_pages = 0; + uint32_t start_index = 0; + uint32_t free_pages = 0; + uint32_t i = index; + + /* Check the continuous pages count is not smaller than unit count */ + for (; free_pages < attr->unit && i < mem->page_count; i++) { + checked_pages++; + state = &mem->states[i]; + if (dca_page_is_allocable(state, free_pages == 0)) { + if (free_pages == 0) + start_index = i; + + free_pages++; + } else { + free_pages = 0; + } + } + + if (free_pages < attr->unit) + return DCA_MEM_NEXT_ITERATE; + + for (i = 0; i < free_pages; i++) { + state = &mem->states[start_index + i]; + lock_dca_page_to_attach(state, attr->buf_id); + attr->total++; + } + + if (attr->total >= attr->max) + return DCA_MEM_STOP_ITERATE; + + return checked_pages; +} + +static uint32_t assign_dca_pages(struct udma_dca_ctx *ctx, uint32_t buf_id, + uint32_t count, uint32_t unit) +{ + struct dca_page_assign_attr attr = {}; + + attr.buf_id = buf_id; + attr.unit = unit; + attr.max = count; + travel_dca_pages(ctx, &attr, assign_dca_pages_proc); + return attr.total; +} + +static int clear_dca_pages_proc(struct dca_mem *mem, uint32_t index, + void *param) +{ + struct dca_page_state *state = &mem->states[index]; + struct dca_page_clear_attr *attr = param; + + if (dca_page_is_attached(state, attr->buf_id)) { + set_dca_page_to_free(state); + attr->clear_pages++; + } + + if (attr->clear_pages >= attr->max_pages) + return DCA_MEM_STOP_ITERATE; + else + return 0; +} + +static void clear_dca_pages(struct udma_dca_ctx *ctx, uint32_t buf_id, + uint32_t count) +{ + struct dca_page_clear_attr attr = {}; + + attr.buf_id = buf_id; + attr.max_pages = count; + travel_dca_pages(ctx, &attr, clear_dca_pages_proc); +} + +static uint32_t alloc_buf_from_dca_mem(struct udma_qp *qp, + struct udma_dca_ctx *ctx) +{ + uint32_t alloc_pages; + uint32_t unit_pages; + uint32_t buf_pages; + uint32_t buf_id; + + buf_pages = qp->dca_cfg.npages; + /* Gen new buf id */ + buf_id = UDMA_DCA_TO_BUF_ID(qp->qpn, qp->dca_cfg.attach_count); + + /* Assign pages from free pages */ + unit_pages = qp->mtr.hem_cfg.is_direct ? buf_pages : 1; + alloc_pages = assign_dca_pages(ctx, buf_id, buf_pages, unit_pages); + if (buf_pages != alloc_pages) { + if (alloc_pages > 0) + clear_dca_pages(ctx, buf_id, alloc_pages); + return UDMA_DCA_INVALID_BUF_ID; + } + + return buf_id; +} + +static int sync_dca_buf_offset(struct udma_dev *dev, struct udma_qp *qp, + struct udma_dca_attach_attr *attr) +{ + if (qp->sq.wqe_cnt > 0) { + if (attr->sq_offset >= qp->sge.offset) { + dev_err(dev->dev, "failed to check SQ offset = %u\n", + attr->sq_offset); + return -EINVAL; + } + qp->sq.wqe_offset = qp->sq.offset + attr->sq_offset; + } + + if (qp->sge.sge_cnt > 0) + qp->sge.wqe_offset = qp->sge.offset + attr->sge_offset; + + return 0; +} + +static int get_alloced_umem_proc(struct dca_mem *mem, uint32_t index, + void *param) +{ + struct dca_get_alloced_pages_attr *attr = param; + struct dca_page_state *states = mem->states; + struct ubcore_umem *umem = mem->pages; + uint32_t npage_per_sg, k, i; + struct scatterlist *sg; + + for_each_sg(umem->sg_head.sgl, sg, umem->sg_head.nents, k) { + npage_per_sg = sg_dma_len(sg) >> UDMA_HW_PAGE_SHIFT; + for (i = 0; i < npage_per_sg; ++i) { + if (dca_page_is_allocated(&states[i], attr->buf_id)) { + attr->pages[attr->total++] = sg_dma_address(sg) + + (i << UDMA_HW_PAGE_SHIFT); + } + if (attr->total >= attr->max) + return DCA_MEM_STOP_ITERATE; + } + } + + return DCA_MEM_NEXT_ITERATE; +} + +static int config_dca_qpc(struct udma_dev *dev, struct udma_qp *qp, + dma_addr_t *pages, uint32_t page_count) +{ + int ret; + + ret = udma_mtr_map(dev, &qp->mtr, pages, page_count); + if (ret) { + dev_err(dev->dev, "failed to map DCA pages, ret = %d.\n", ret); + return ret; + } + + ret = udma_set_dca_buf(dev, qp); + if (ret) { + dev_err(dev->dev, "failed to set DCA to HW, ret = %d.\n", ret); + return ret; + } + + return 0; +} + +static int setup_dca_buf_to_hw(struct udma_dev *dev, struct udma_qp *qp, + struct udma_dca_ctx *ctx, uint32_t buf_id, + uint32_t count) +{ + struct dca_get_alloced_pages_attr attr = {}; + dma_addr_t *pages; + int ret; + + /* alloc a tmp array to store buffer's dma address */ + pages = kvcalloc(count, sizeof(dma_addr_t), GFP_ATOMIC); + if (!pages) + return -ENOMEM; + + attr.buf_id = buf_id; + attr.pages = pages; + attr.max = count; + + travel_dca_pages(ctx, &attr, get_alloced_umem_proc); + + if (attr.total != count) { + dev_err(dev->dev, "failed to get DCA page %u != %u.\n", + attr.total, count); + ret = -ENOMEM; + goto err_get_pages; + } + + ret = config_dca_qpc(dev, qp, pages, count); +err_get_pages: + /* drop tmp array */ + kvfree(pages); + + return ret; +} + +static int active_dca_pages_proc(struct dca_mem *mem, uint32_t index, + void *param) +{ + struct dca_page_active_attr *attr = param; + struct dca_page_state *state; + bool changed = false; + bool stop = false; + int i, free_pages; + + free_pages = 0; + for (i = 0; !stop && i < mem->page_count; i++) { + state = &mem->states[i]; + if (dca_page_is_free(state)) { + free_pages++; + } else if (dca_page_is_allocated(state, attr->buf_id)) { + free_pages++; + /* Change matched pages state */ + unlock_dca_page_to_active(state, attr->buf_id); + changed = true; + attr->alloc_pages++; + if (attr->alloc_pages == attr->max_pages) + stop = true; + } + } + + for (; changed && i < mem->page_count; i++) + if (dca_page_is_free(state)) + free_pages++; + + /* Clean mem changed to dirty */ + if (changed && free_pages == mem->page_count) + attr->dirty_mems++; + + return stop ? DCA_MEM_STOP_ITERATE : DCA_MEM_NEXT_ITERATE; +} + +static uint32_t active_dca_pages(struct udma_dca_ctx *ctx, uint32_t buf_id, + uint32_t count) +{ + struct dca_page_active_attr attr = {}; + unsigned long flags; + + attr.buf_id = buf_id; + attr.max_pages = count; + travel_dca_pages(ctx, &attr, active_dca_pages_proc); + + /* Update free size */ + spin_lock_irqsave(&ctx->pool_lock, flags); + ctx->free_mems -= attr.dirty_mems; + ctx->free_size -= attr.alloc_pages << UDMA_HW_PAGE_SHIFT; + spin_unlock_irqrestore(&ctx->pool_lock, flags); + + return attr.alloc_pages; +} + +static int active_alloced_buf(struct udma_dev *dev, struct udma_dca_ctx *ctx, + struct udma_qp *qp, uint32_t buf_id, + struct udma_dca_attach_attr *attr) +{ + uint32_t active_pages; + uint32_t alloc_pages; + int ret; + + alloc_pages = qp->dca_cfg.npages; + ret = sync_dca_buf_offset(dev, qp, attr); + if (ret) { + dev_err(dev->dev, "failed to sync DCA offset, ret = %d\n", ret); + goto active_fail; + } + + ret = setup_dca_buf_to_hw(dev, qp, ctx, buf_id, alloc_pages); + if (ret) { + dev_err(dev->dev, "failed to setup DCA buf, ret = %d.\n", ret); + goto active_fail; + } + + active_pages = active_dca_pages(ctx, buf_id, alloc_pages); + if (active_pages != alloc_pages) { + dev_err(dev->dev, "failed to active DCA pages, %u != %u.\n", + active_pages, alloc_pages); + ret = -ENOBUFS; + goto active_fail; + } + + return 0; +active_fail: + clear_dca_pages(ctx, buf_id, alloc_pages); + return ret; +} + +int udma_dca_attach(struct udma_dev *dev, struct udma_dca_attach_attr *attr, + struct udma_dca_attach_resp *resp) +{ + struct udma_dca_ctx *ctx; + struct udma_dca_cfg *cfg; + struct udma_qp *qp; + uint32_t buf_id; + int ret; + + qp = get_qp(dev, attr->qpn); + if (qp == NULL) { + dev_err(dev->dev, "failed to find attach qp, qpn = 0x%llx\n", + attr->qpn); + return -EINVAL; + } + cfg = &qp->dca_cfg; + ctx = qp->dca_ctx; + + stop_aging_dca_mem(ctx, cfg, false); + resp->alloc_flags = 0; + + spin_lock(&cfg->lock); + buf_id = cfg->buf_id; + /* Already attached */ + if (buf_id != UDMA_DCA_INVALID_BUF_ID) { + resp->alloc_pages = cfg->npages; + spin_unlock(&cfg->lock); + return 0; + } + + /* Start to new attach */ + resp->alloc_pages = 0; + buf_id = alloc_buf_from_dca_mem(qp, ctx); + if (buf_id == UDMA_DCA_INVALID_BUF_ID) { + spin_unlock(&cfg->lock); + /* No report fail, need try again after the pool increased */ + return 0; + } + + ret = active_alloced_buf(dev, ctx, qp, buf_id, attr); + if (ret) { + spin_unlock(&cfg->lock); + dev_err(dev->dev, + "failed to active DCA buf for QP-%llu, ret = %d.\n", + qp->qpn, ret); + return ret; + } + + /* Attach ok */ + cfg->buf_id = buf_id; + cfg->attach_count++; + spin_unlock(&cfg->lock); + + resp->alloc_flags |= UDMA_DCA_ATTACH_FLAGS_NEW_BUFFER; + resp->alloc_pages = cfg->npages; + resp->dcan = cfg->dcan; + update_dca_buf_status(ctx, cfg->dcan, true); + + return 0; +} + +void udma_dca_disattach(struct udma_dev *dev, struct udma_dca_attach_attr *attr) +{ + struct udma_dca_ctx *ctx; + struct udma_dca_cfg *cfg; + struct udma_qp *qp; + + qp = get_qp(dev, attr->qpn); + if (qp == NULL) { + dev_err(dev->dev, "failed to find disattach qp, qpn = 0x%llx\n", + attr->qpn); + return; + } + cfg = &qp->dca_cfg; + ctx = qp->dca_ctx; + + clear_dca_pages(ctx, cfg->buf_id, qp->dca_cfg.npages); + + cfg->buf_id = UDMA_DCA_INVALID_BUF_ID; + + update_dca_buf_status(ctx, cfg->dcan, false); +} + +void udma_dca_detach(struct udma_dev *dev, struct udma_dca_detach_attr *attr) +{ + struct udma_dca_ctx *ctx; + struct udma_dca_cfg *cfg; + struct udma_qp *qp; + + qp = get_qp(dev, attr->qpn); + if (qp == NULL) { + dev_err(dev->dev, "failed to find detach qp, qpn = 0x%llx\n", + attr->qpn); + return; + } + cfg = &qp->dca_cfg; + ctx = qp->dca_ctx; + + stop_aging_dca_mem(ctx, cfg, true); + + spin_lock(&ctx->aging_lock); + spin_lock(&cfg->lock); + cfg->sq_idx = attr->sq_idx; + list_add_tail(&cfg->aging_node, &ctx->aging_new_list); + spin_unlock(&cfg->lock); + spin_unlock(&ctx->aging_lock); + + restart_aging_dca_mem(dev, ctx); +} + +static int enum_dca_pool_proc(struct dca_mem *mem, uint32_t index, void *param) +{ + struct dca_mem_enum_attr *attr = param; + int ret; + + ret = attr->enum_fn((struct dca_page_state *)(mem->states), + mem->page_count, attr->param); + + return ret ? DCA_MEM_STOP_ITERATE : DCA_MEM_NEXT_ITERATE; +} + +void udma_enum_dca_pool(struct udma_dca_ctx *dca_ctx, void *param, + udma_dca_enum_callback cb) +{ + struct dca_mem_enum_attr attr; + + attr.enum_fn = cb; + attr.param = param; + travel_dca_pages(dca_ctx, &attr, enum_dca_pool_proc); +} diff --git a/drivers/ub/hw/hns3/hns3_udma_dca.h b/drivers/ub/hw/hns3/hns3_udma_dca.h new file mode 100644 index 0000000000000000000000000000000000000000..09cb906c96a2a101c5fa8de9f5d49687b1ef4b50 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_dca.h @@ -0,0 +1,219 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_DCA_H +#define _UDMA_DCA_H + +#include "hns3_udma_abi.h" +#include "hns3_udma_tp.h" +#include "hns3_udma_hem.h" + +#define DCA_MEM_FLAGS_ALLOCED BIT(0) +#define DCA_MEM_FLAGS_REGISTERED BIT(1) + +#define DCA_MEM_AGEING_MSES 1000 /* DCA mem ageing interval time */ +#define UDMA_DCA_INVALID_BUF_ID 0U +#define DCA_MEM_STOP_ITERATE (-1) +#define DCA_MEM_NEXT_ITERATE (-2) + +#define DCAN_TO_SYNC_BIT(n) ((n) * UDMA_DCA_BITS_PER_STATUS) +#define DCAN_TO_STAT_BIT(n) DCAN_TO_SYNC_BIT(n) + +#define UDMA_DCA_OWN_MASK GENMASK(21, 0) + +/* + * buffer id(29b) = tag(7b) + owner(22b) + * [28:22] tag : indicate the QP config update times. + * [21: 0] owner: indicate the QP to which the page belongs. + */ +#define UDMA_DCA_ID_MASK GENMASK(28, 0) +#define UDMA_DCA_TAG_MASK GENMASK(28, 22) +#define UDMA_DCA_OWN_MASK GENMASK(21, 0) + +#define UDMA_DCA_BUF_ID_TO_TAG(buf_id) (((buf_id) & UDMA_DCA_TAG_MASK) >> 22) +#define UDMA_DCA_BUF_ID_TO_QPN(buf_id) ((buf_id) & UDMA_DCA_OWN_MASK) +#define UDMA_DCA_TO_BUF_ID(qpn, tag) (((qpn) & UDMA_DCA_OWN_MASK) | \ + (((tag) << 22) & UDMA_DCA_TAG_MASK)) + +/* DCA page state (32 bit) */ +struct dca_page_state { + uint32_t buf_id : 29; /* If zero, means page can be used by any buffer. */ + uint32_t lock : 1; /* @buf_id locked this page to prepare access. */ + uint32_t active : 1; /* @buf_id is accessing this page. */ + uint32_t head : 1; /* This page is the head in a continuous address range. */ +}; + +struct dca_mem { + uint32_t flags; + struct list_head list; /* link to mem list in dca context */ + spinlock_t lock; /* protect the @flags and @list */ + uint32_t page_count; /* page count in this mem obj */ + uint64_t key; /* register by caller */ + uint32_t size; /* bytes in this mem object */ + struct dca_page_state *states; /* record each page's state */ + void *pages; /* memory handle for getting dma address */ +}; + +struct dca_page_free_buf_attr { + uint32_t buf_id; + uint32_t max_pages; + uint32_t free_pages; + uint32_t clean_mems; +}; + +struct dca_page_assign_attr { + uint32_t buf_id; + uint32_t unit; + uint32_t total; + uint32_t max; +}; + +struct dca_page_clear_attr { + uint32_t buf_id; + uint32_t max_pages; + uint32_t clear_pages; +}; + +struct dca_get_alloced_pages_attr { + uint32_t buf_id; + dma_addr_t *pages; + uint32_t total; + uint32_t max; +}; + +struct dca_page_active_attr { + uint32_t buf_id; + uint32_t max_pages; + uint32_t alloc_pages; + uint32_t dirty_mems; +}; + +struct dca_page_query_active_attr { + uint32_t buf_id; + uint32_t curr_index; + uint32_t start_index; + uint32_t page_index; + uint32_t page_count; + uint64_t mem_key; +}; + +typedef int (*udma_dca_enum_callback)(struct dca_page_state *states, + uint32_t count, void *param); + +struct dca_mem_enum_attr { + void *param; + udma_dca_enum_callback enum_fn; +}; + +static inline uint64_t umem_cal_npages(uint64_t va, uint64_t len) +{ + return (ALIGN(va + len, UDMA_PAGE_SIZE) - ALIGN_DOWN(va, UDMA_PAGE_SIZE)) / + UDMA_PAGE_SIZE; +} + +static inline bool dca_page_is_attached(struct dca_page_state *state, + uint32_t buf_id) +{ + /* only the own bit needs to be matched. */ + return (UDMA_DCA_OWN_MASK & buf_id) == + (UDMA_DCA_OWN_MASK & state->buf_id); +} + +static inline bool dca_mem_is_available(struct dca_mem *mem) +{ + return mem->flags == (DCA_MEM_FLAGS_ALLOCED | DCA_MEM_FLAGS_REGISTERED); +} + +static inline void set_dca_page_to_free(struct dca_page_state *state) +{ + state->buf_id = UDMA_DCA_INVALID_BUF_ID; + state->active = 0; + state->lock = 0; +} + +static inline bool dca_page_is_free(struct dca_page_state *state) +{ + return state->buf_id == UDMA_DCA_INVALID_BUF_ID; +} + +static inline bool dca_page_is_active(struct dca_page_state *state, + uint32_t buf_id) +{ + /* all buf id bits must be matched */ + return (UDMA_DCA_ID_MASK & buf_id) == state->buf_id && + !state->lock && state->active; +} + +static inline bool dca_page_is_inactive(struct dca_page_state *state) +{ + return !state->lock && !state->active; +} + +static inline void lock_dca_page_to_attach(struct dca_page_state *state, + uint32_t buf_id) +{ + state->buf_id = UDMA_DCA_ID_MASK & buf_id; + state->active = 0; + state->lock = 1; +} + +static inline bool dca_page_is_allocated(struct dca_page_state *state, + uint32_t buf_id) +{ + return dca_page_is_attached(state, buf_id) && state->lock; +} + +static inline void unlock_dca_page_to_active(struct dca_page_state *state, + uint32_t buf_id) +{ + state->buf_id = UDMA_DCA_ID_MASK & buf_id; + state->active = 1; + state->lock = 0; +} + +void udma_enable_dca(struct udma_dev *dev, struct udma_qp *qp); +void udma_disable_dca(struct udma_dev *dev, struct udma_qp *qp); + +void udma_modify_dca(struct udma_dev *dev, struct udma_qp *qp); + +int udma_register_dca_mem(struct udma_dev *dev, struct udma_ucontext *context, + struct udma_dca_reg_attr *attr); +void unregister_dca_mem(struct udma_dev *dev, struct udma_dca_ctx *ctx, + struct dca_mem *mem); +int udma_deregister_dca_mem(struct udma_dev *dev, + struct udma_ucontext *context, + struct udma_dca_dereg_attr *attr, bool from_user); + +void udma_shrink_dca_mem(struct udma_dev *dev, struct udma_ucontext *context, + struct udma_dca_shrink_attr *attr, + struct udma_dca_shrink_resp *resp); + +int udma_query_dca_mem(struct udma_dev *dev, struct udma_dca_query_attr *attr, + struct udma_dca_query_resp *resp); + +int udma_dca_attach(struct udma_dev *dev, struct udma_dca_attach_attr *attr, + struct udma_dca_attach_resp *resp); +void udma_dca_disattach(struct udma_dev *dev, struct udma_dca_attach_attr *attr); +void udma_dca_detach(struct udma_dev *dev, struct udma_dca_detach_attr *attr); + +int udma_register_udca(struct udma_dev *udma_dev, + struct udma_ucontext *context, struct ubcore_udrv_priv *udrv_data); + +void udma_unregister_udca(struct udma_dev *udma_dev, + struct udma_ucontext *context); + +void udma_enum_dca_pool(struct udma_dca_ctx *dca_ctx, void *param, + udma_dca_enum_callback cb); +#endif diff --git a/drivers/ub/hw/hns3/hns3_udma_device.h b/drivers/ub/hw/hns3/hns3_udma_device.h new file mode 100644 index 0000000000000000000000000000000000000000..78e03558fe7516e8e789ec5a9ee73945c9f7c42d --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_device.h @@ -0,0 +1,969 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_DEVICE_H +#define _UDMA_DEVICE_H + +#include +#include +#include "hns3_udma_hw.h" +#include "hns3_udma_common.h" + +#define UDMA_MAX_PORTS 6 + +#define BA_BYTE_LEN 8 + +#define UDMA_INVALID_ID 0xffff + +/* Configure to HW for PAGE_SIZE larger than 4KB */ +#define PG_SHIFT_OFFSET (PAGE_SHIFT - 12) + +#define CQ_BANKID_SHIFT 2 + +#define UDMA_MAX_IRQ_NUM 128 + +#define MTT_MIN_COUNT 2 + +#define UDMA_QP_BANK_NUM 8 +#define QP_BANKID_SHIFT 3 +#define QP_BANKID_MASK GENMASK(2, 0) +#define UDMA_QPC_SZ 512 +#define UDMA_CQE_SZ 64 +#define UDMA_SCCC_SZ 64 +#define UDMA_GMV_ENTRY_SZ 32 + +#define UDMA_CQ_BANK_NUM 4 + +#define UDMA_SGE_IN_WQE 2 +#define UDMA_SGE_SHIFT 4 +#define UDMA_SGE_SIZE 16 +#define UDMA_IDX_QUE_ENTRY_SZ 4 +/* The minimum page size is 4K for hardware */ +#define UDMA_HW_PAGE_SHIFT 12 +#define UDMA_PAGE_SIZE (1 << UDMA_HW_PAGE_SHIFT) +#define udma_hw_page_align(x) ALIGN(x, 1 << UDMA_HW_PAGE_SHIFT) + +#define UDMA_DWQE_SIZE 65536 +#define UDMA_DWQE_MMAP_QP_NUM 1024 + +#define UDMA_HOP_NUM_0 0xff +#define UDMA_CAP_FLAGS_EX_SHIFT 12 + +#define UDMA_CMQ_TX_TIMEOUT 30000 +#define UDMA_CMQ_DESC_NUM_S 3 +#define UDMA_CMD_CSQ_DESC_NUM 1024 + +#define UDMA_TX_CMQ_BASEADDR_L_REG 0x07000 +#define UDMA_TX_CMQ_BASEADDR_H_REG 0x07004 +#define UDMA_TX_CMQ_DEPTH_REG 0x07008 +#define UDMA_TX_CMQ_PI_REG 0x07010 +#define UDMA_TX_CMQ_CI_REG 0x07014 + +#define UDMA_MAX_MSG_LEN 0x80000000 + +#define UDMA_MAX_BT_REGION 3 +#define UDMA_MAX_BT_LEVEL 3 + +#define CQC_FIELD_LOC(h, l) ((uint64_t)(h) << 32 | (l)) + +#define CQC_CQE_BA_L_OFFSET 3 +#define CQC_CQE_BA_H_OFFSET 35 + +#define CQC_CQ_ST CQC_FIELD_LOC(1, 0) +#define CQC_NOTIFY_MODE CQC_FIELD_LOC(4, 4) +#define CQC_ARM_ST CQC_FIELD_LOC(7, 6) +#define CQC_SHIFT CQC_FIELD_LOC(12, 8) +#define CQC_NOTIFY_ADDR_0 CQC_FIELD_LOC(31, 24) +#define CQC_CQN CQC_FIELD_LOC(55, 32) +#define CQC_POE_EN CQC_FIELD_LOC(56, 56) +#define CQC_POE_NUM CQC_FIELD_LOC(58, 57) +#define CQC_CQE_SIZE CQC_FIELD_LOC(60, 59) +#define CQC_NOTIFY_DEVICE_EN CQC_FIELD_LOC(62, 62) +#define CQC_CQE_CUR_BLK_ADDR_L CQC_FIELD_LOC(95, 64) +#define CQC_CQE_CUR_BLK_ADDR_H CQC_FIELD_LOC(115, 96) +#define CQC_POE_QID CQC_FIELD_LOC(125, 116) +#define CQC_CQE_HOP_NUM CQC_FIELD_LOC(127, 126) +#define CQC_CQE_NEX_BLK_ADDR_L CQC_FIELD_LOC(159, 128) +#define CQC_CQE_NEX_BLK_ADDR_H CQC_FIELD_LOC(179, 160) +#define CQC_NOTIFY_ADDR_2 CQC_FIELD_LOC(183, 180) +#define CQC_CQE_BAR_PG_SZ CQC_FIELD_LOC(187, 184) +#define CQC_CQE_BUF_PG_SZ CQC_FIELD_LOC(191, 188) +#define CQC_NOTIFY_ADDR_3 CQC_FIELD_LOC(223, 216) +#define CQC_NOTIFY_ADDR_4 CQC_FIELD_LOC(255, 248) +#define CQC_CQE_BA_L CQC_FIELD_LOC(287, 256) +#define CQC_CQE_BA_H CQC_FIELD_LOC(316, 288) +#define CQC_DB_RECORD_EN CQC_FIELD_LOC(320, 320) +#define CQC_CQE_DB_RECORD_ADDR_L CQC_FIELD_LOC(351, 321) +#define CQC_CQE_DB_RECORD_ADDR_H CQC_FIELD_LOC(383, 352) +#define CQC_CQE_CNT CQC_FIELD_LOC(407, 384) +#define CQC_NOTIFY_ADDR_5 CQC_FIELD_LOC(415, 408) +#define CQC_CQ_MAX_CNT CQC_FIELD_LOC(431, 416) +#define CQC_CQ_PERIOD CQC_FIELD_LOC(447, 432) +#define CQC_NOTIFY_ADDR_6 CQC_FIELD_LOC(509, 504) +#define CQC_NOTIFY_EN CQC_FIELD_LOC(510, 510) + +#define UDMA_CQ_DEFAULT_BURST_NUM 0x0 +#define UDMA_CQ_DEFAULT_INTERVAL 0x0 + +#define EQ_ENABLE 1 +#define EQ_DISABLE 0 +#define UDMA_CEQ 0 +#define UDMA_AEQ 1 +#define UDMA_AEQ_DEFAULT_BURST_NUM 0x0 +#define UDMA_AEQ_DEFAULT_INTERVAL 0x0 +#define UDMA_CEQ_DEFAULT_BURST_NUM 0x0 +#define UDMA_CEQ_DEFAULT_INTERVAL 0x0 +#define UDMA_VF_EQ_DB_CFG0_REG 0x238 +#define UDMA_VF_ABN_INT_CFG_REG 0x13000 +#define UDMA_VF_ABN_INT_ST_REG 0x13004 +#define UDMA_VF_ABN_INT_EN_REG 0x13008 +#define UDMA_VF_EVENT_INT_EN_REG 0x1300c +#define EQ_REG_OFFSET 0x4 +#define MTU_VAL_256 256 +#define MTU_VAL_512 512 +#define MTU_VAL_1024 1024 +#define MTU_VAL_2048 2048 +#define MTU_VAL_4096 4096 +#define UDMA_DEFAULT_MAX_JETTY_X_SHIFT 8 + +#define UDMA_DB_ADDR_OFFSET 0x230 +#define UDMA_DEV_START_OFFSET 2 +#define UDMA_DEV_EX_START_OFFSET 4 + +#define UDMA_MIN_JFS_DEPTH 64 + +#define UDMA_DCA_BITS_PER_STATUS 1 +#define DCA_BITS_HALF 2 + +enum { + NO_ARMED = 0x0 +}; + +enum { + CQE_SIZE_64B = 0x1 +}; + +#define CQ_STATE_VALID 1 + +enum udma_reset_stage { + UDMA_STATE_RST_DOWN = 2, + UDMA_STATE_RST_UNINIT, + UDMA_STATE_RST_INIT, + UDMA_STATE_RST_INITED, +}; + +enum udma_instance_state { + UDMA_STATE_NON_INIT, + UDMA_STATE_INIT, + UDMA_STATE_INITED, + UDMA_STATE_UNINIT, +}; + +#define UDMA_IS_RESETTING 1 + +enum { + UDMA_RST_DIRECT_RETURN = 0, +}; + +enum udma_event { + UDMA_EVENT_TYPE_COMM_EST = 0x03, + UDMA_EVENT_TYPE_WQ_CATAS_ERROR = 0x05, + UDMA_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR = 0x06, + UDMA_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR = 0x07, + UDMA_EVENT_TYPE_JFR_LIMIT_REACH = 0x08, + UDMA_EVENT_TYPE_JFR_LAST_WQE_REACH = 0x09, + UDMA_EVENT_TYPE_JFC_ACCESS_ERROR = 0x0b, + UDMA_EVENT_TYPE_JFC_OVERFLOW = 0x0c, + UDMA_EVENT_TYPE_MB = 0x13, +}; + +enum udma_mtu { + UDMA_MTU_256 = 1, + UDMA_MTU_512 = 2, + UDMA_MTU_1024 = 3, + UDMA_MTU_2048 = 4, + UDMA_MTU_4096 = 5 +}; + +enum { + /* discard BIT(2), reserved for compatibility */ + UDMA_CAP_FLAG_CQ_RECORD_DB = BIT(3), + UDMA_CAP_FLAG_QP_RECORD_DB = BIT(4), + UDMA_CAP_FLAG_SRQ = BIT(5), + UDMA_CAP_FLAG_QP_FLOW_CTRL = BIT(9), + UDMA_CAP_FLAG_DIRECT_WQE = BIT(12), + UDMA_CAP_FLAG_SDI_MODE = BIT(14), + UDMA_CAP_FLAG_DCA_MODE = BIT(15), + UDMA_CAP_FLAG_WRITE_NOTIFY = BIT(16), + UDMA_CAP_FLAG_STASH = BIT(17), + UDMA_CAP_FLAG_CQE_INLINE = BIT(19), + UDMA_CAP_FLAG_SRQ_RECORD_DB = BIT(22), + UDMA_CAP_FLAG_OOR = BIT(25), + UDMA_CAP_FLAG_AR = BIT(26), + UDMA_CAP_FLAG_POE = BIT(27), +}; + +enum { + TYPE_CSQ = 1 +}; + +enum udma_cong_type { + UDMA_CONG_TYPE_DCQCN, + UDMA_CONG_TYPE_LDCP, + UDMA_CONG_TYPE_HC3, + UDMA_CONG_TYPE_DIP, + UDMA_CONG_TYPE_TOTAL, +}; + +enum udma_cong_sel { + UDMA_CONG_SEL_DCQCN = 1 << UDMA_CONG_TYPE_DCQCN, + UDMA_CONG_SEL_LDCP = 1 << UDMA_CONG_TYPE_LDCP, + UDMA_CONG_SEL_HC3 = 1 << UDMA_CONG_TYPE_HC3, + UDMA_CONG_SEL_DIP = 1 << UDMA_CONG_TYPE_DIP, +}; + +enum udma_sig_type { + SIGNAL_REQ_WR = 1, +}; + +enum udma_qp_type { + QPT_RC, + QPT_UD = 0x3, +}; + +enum udma_qp_state { + QPS_RESET, + QPS_RTR = 2, + QPS_RTS, + QPS_ERR = 6, +}; + +enum udma_eq_dfx { + UDMA_DFX_AEQE, + UDMA_DFX_CEQE, + UDMA_DFX_EQ_TOTAL +}; + +enum { + UDMA_BUF_DIRECT = BIT(0), + UDMA_BUF_NOSLEEP = BIT(1), + UDMA_BUF_NOFAIL = BIT(2), +}; + +static inline enum ubcore_mtu udma_mtu_int_to_enum(int mtu) +{ + if (mtu >= MTU_VAL_4096) + return UBCORE_MTU_4096; + else if (mtu >= MTU_VAL_2048) + return UBCORE_MTU_2048; + else if (mtu >= MTU_VAL_1024) + return UBCORE_MTU_1024; + else if (mtu >= MTU_VAL_512) + return UBCORE_MTU_512; + else + return UBCORE_MTU_256; +} + +struct udma_uar { + uint64_t pfn; + uint64_t index; + uint64_t logic_idx; +}; + +struct udma_ida { + struct ida ida; + uint32_t min; /* Lowest ID to allocate. */ + uint32_t max; /* Highest ID to allocate. */ +}; + +struct udma_buf_region { + uint32_t offset; /* page offset */ + uint32_t count; /* page count */ + int hopnum; /* addressing hop num */ +}; + +struct udma_hem_list { + struct list_head root_bt; + /* link all bt dma mem by hop config */ + struct list_head mid_bt[UDMA_MAX_BT_REGION][UDMA_MAX_BT_LEVEL]; + struct list_head btm_bt; /* link all bottom bt in @mid_bt */ + dma_addr_t root_ba; /* pointer to the root ba table */ +}; + +struct udma_buf_attr { + struct { + size_t size; /* region size */ + int hopnum; /* multi-hop addressing hop num */ + } region[UDMA_MAX_BT_REGION]; + uint32_t region_count; /* valid region count */ + uint32_t page_shift; /* buffer page shift */ + /* only alloc buffer-required MTT memory */ + bool mtt_only; +}; + +struct udma_buf_list { + void *buf; + dma_addr_t map; +}; + +struct udma_hem_cfg { + dma_addr_t root_ba; /* root BA table's address */ + bool is_direct; /* addressing without BA table */ + uint32_t ba_pg_shift; /* BA table page shift */ + uint32_t buf_pg_shift; /* buffer page shift */ + uint32_t buf_pg_count; /* buffer page count */ + struct udma_buf_region region[UDMA_MAX_BT_REGION]; + uint32_t region_count; +}; + +struct udma_buf { + struct udma_buf_list *trunk_list; + uint32_t ntrunks; + uint32_t npages; + uint32_t trunk_shift; + uint32_t page_shift; +}; + +struct udma_link_table { + struct udma_buf_list table; + struct udma_buf *buf; +}; + +/* memory translate region */ +struct udma_mtr { + struct udma_hem_list hem_list; /* multi-hop addressing resource */ + struct ubcore_umem *umem; /* user space buffer */ + struct udma_buf *kmem; /* kernel space buffer */ + struct udma_hem_cfg hem_cfg; /* config for hardware addressing */ +}; + +struct udma_user_db_page { + struct list_head list; + struct ubcore_umem *umem; + uint64_t user_virt; + refcount_t refcount; +}; + +struct udma_ceqe { + uint32_t comp; + uint32_t rsv[15]; +}; + +struct udma_aeqe { + uint32_t asyn; + union { + struct { + uint32_t num; + uint32_t rsv0; + uint32_t rsv1; + } queue_event; + + struct { + uint64_t out_param; + uint16_t token; + uint8_t status; + uint8_t rsv0; + } __packed cmd; + } event; + uint32_t rsv[12]; +}; + +struct udma_work { + struct udma_dev *udma_dev; + struct work_struct work; + int event_type; + int sub_type; + struct udma_aeqe aeqe; + uint32_t queue_num; + uint32_t eq_ci; + int eqn; +}; + +struct udma_dev; + +struct udma_db { + uint32_t *db_record; + struct udma_user_db_page *user_page; + dma_addr_t dma; + void *virt_addr; +}; + +struct udma_dca_ctx { + struct list_head pool; /* all DCA mems link to @pool */ + spinlock_t pool_lock; /* protect @pool */ + uint32_t free_mems; /* free mem num in pool */ + size_t free_size; /* free mem size in pool */ + size_t total_size; /* total size in pool */ + size_t max_size; /* max size the pool can expand to */ + size_t min_size; /* shrink if @free_size > @min_size */ + uint32_t unit_size; /* unit size per DCA mem */ + + uint32_t max_qps; + uint32_t status_npage; + struct ida ida; + + uintptr_t *buf_status; + uintptr_t *sync_status; + + bool exit_aging; + struct list_head aging_proc_list; + struct list_head aging_new_list; + spinlock_t aging_lock; + struct delayed_work aging_dwork; +}; + +struct udma_ucontext { + struct ubcore_ucontext uctx; + struct udma_uar uar; + uint64_t pdn; + struct udma_dca_ctx dca_ctx; + void *dca_dbgfs; +}; + +struct udma_cmd_context { + struct completion done; + int result; + int next; + uint64_t out_param; + uint16_t token; + uint16_t busy; +}; + +struct udma_cmq_desc { + uint16_t opcode; + uint16_t flag; + uint16_t retval; + uint16_t rsv; + uint32_t data[6]; +}; + +enum udma_cmdq_state { + UDMA_CMDQ_STATE_HEAD_TAIL_ERR = 1, + UDMA_CMDQ_STATE_FATAL_ERR, +}; + +struct udma_cmq_ring { + dma_addr_t desc_dma_addr; + struct udma_cmq_desc *desc; + uint32_t head; + uint16_t desc_num; + uint8_t flag; + struct mutex lock; +}; + +struct udma_cmdq { + struct dma_pool *pool; + struct semaphore poll_sem; + struct semaphore event_sem; + int max_cmds; + spinlock_t ctx_lock; + int free_head; + struct udma_cmd_context *context; + /* + * Process whether use event mode, init default non-zero + * After the event queue of cmd event ready, + * can switch into event mode + * close device, switch into poll mode(non event mode) + */ + uint8_t use_events; + struct rw_semaphore udma_mb_rwsem; + enum udma_cmdq_state state; +}; + +struct udma_cmd_mailbox { + void *buf; + dma_addr_t dma; +}; + +struct udma_netdev { + spinlock_t lock; + struct net_device *netdevs[UDMA_MAX_PORTS]; +}; + +enum udma_device_state { + UDMA_DEVICE_STATE_RST_DOWN = 1, + UDMA_DEVICE_STATE_UNINIT, +}; + +struct udma_reset_state { + uint32_t reset_state; /* stored to use in user space */ +}; + +struct udma_cmq { + struct udma_cmq_ring csq; + uint16_t tx_timeout; +}; + +struct udma_priv { + struct hnae3_handle *handle; + struct udma_cmq cmq; + struct udma_link_table ext_llm; +}; + +struct udma_hem_table { + /* HEM type: 0 = qpc, 1 = mtt, 2 = cqc, 3 = srq, 4 = other */ + uint32_t type; + /* HEM array elment num */ + uint64_t num_hem; + /* Single obj size */ + uint64_t obj_size; + uint64_t table_chunk_size; + struct mutex mutex; + struct udma_hem **hem; + uint64_t **bt_l1; + dma_addr_t *bt_l1_dma_addr; + uint64_t **bt_l0; + dma_addr_t *bt_l0_dma_addr; +}; + +struct udma_hw { + int (*cmq_init)(struct udma_dev *udma_dev); + void (*cmq_exit)(struct udma_dev *udma_dev); + int (*hw_profile)(struct udma_dev *udma_dev); + int (*hw_init)(struct udma_dev *udma_dev); + void (*hw_exit)(struct udma_dev *udma_dev); + int (*post_mbox)(struct udma_dev *udma_dev, struct udma_cmq_desc *desc, + uint16_t token, int event); + int (*poll_mbox_done)(struct udma_dev *udma_dev, + uint32_t timeout); + bool (*chk_mbox_avail)(struct udma_dev *udma_dev, bool *is_busy); + int (*set_hem)(struct udma_dev *udma_dev, + struct udma_hem_table *table, int obj, int step_idx); + int (*clear_hem)(struct udma_dev *udma_dev, + struct udma_hem_table *table, int obj, + int step_idx); + int (*set_eid)(struct udma_dev *udma_dev, union ubcore_eid eid); + int (*init_eq)(struct udma_dev *udma_dev); + void (*cleanup_eq)(struct udma_dev *udma_dev); +}; + +struct udma_caps { + uint64_t fw_ver; + uint8_t num_ports; + int gid_table_len[UDMA_MAX_PORTS]; + int pkey_table_len[UDMA_MAX_PORTS]; + int local_ca_ack_delay; + int num_uars; + uint32_t phy_num_uars; + uint32_t max_sq_sg; + uint32_t max_sq_inline; + uint32_t max_rq_sg; + uint32_t max_extend_sg; + uint32_t num_qps; + uint32_t num_qps_shift; + uint32_t num_pi_qps; + uint32_t reserved_qps; + int num_qpc_timer; + int num_srqs; + uint32_t max_wqes; + uint32_t max_srq_wrs; + uint32_t max_srq_sges; + uint32_t max_sq_desc_sz; + uint32_t max_rq_desc_sz; + uint32_t max_srq_desc_sz; + int max_qp_init_rdma; + int max_qp_dest_rdma; + uint32_t num_cqs; + uint32_t max_cqes; + uint32_t min_cqes; + uint32_t min_wqes; + uint32_t reserved_cqs; + int reserved_srqs; + int num_aeq_vectors; + int num_comp_vectors; + int num_other_vectors; + uint32_t num_mtpts; + uint32_t num_mtt_segs; + uint32_t num_srqwqe_segs; + uint32_t num_idx_segs; + int reserved_mrws; + int reserved_uars; + int num_pds; + int reserved_pds; + uint32_t num_xrcds; + uint32_t reserved_xrcds; + uint32_t mtt_entry_sz; + uint32_t cqe_sz; + uint32_t page_size_cap; + uint32_t reserved_lkey; + int mtpt_entry_sz; + int qpc_sz; + int irrl_entry_sz; + int trrl_entry_sz; + int cqc_entry_sz; + int scc_ctx_sz; + int qpc_timer_entry_sz; + int cqc_timer_entry_sz; + int srqc_entry_sz; + int idx_entry_sz; + uint32_t pbl_ba_pg_sz; + uint32_t pbl_buf_pg_sz; + uint32_t pbl_hop_num; + int aeqe_depth; + int ceqe_depth; + uint32_t aeqe_size; + uint32_t ceqe_size; + enum ubcore_mtu max_mtu; + uint32_t qpc_bt_num; + uint32_t qpc_timer_bt_num; + uint32_t srqc_bt_num; + uint32_t cqc_bt_num; + uint32_t cqc_timer_bt_num; + uint32_t mpt_bt_num; + uint32_t eqc_bt_num; + uint32_t smac_bt_num; + uint32_t sgid_bt_num; + uint32_t sccc_bt_num; + uint32_t gmv_bt_num; + uint32_t qpc_ba_pg_sz; + uint32_t qpc_buf_pg_sz; + uint32_t qpc_hop_num; + uint32_t srqc_ba_pg_sz; + uint32_t srqc_buf_pg_sz; + uint32_t srqc_hop_num; + uint32_t cqc_ba_pg_sz; + uint32_t cqc_buf_pg_sz; + uint32_t cqc_hop_num; + uint32_t mpt_ba_pg_sz; + uint32_t mpt_buf_pg_sz; + uint32_t mpt_hop_num; + uint32_t mtt_ba_pg_sz; + uint32_t mtt_buf_pg_sz; + uint32_t mtt_hop_num; + uint32_t wqe_sq_hop_num; + uint32_t wqe_sge_hop_num; + uint32_t wqe_rq_hop_num; + uint32_t sccc_ba_pg_sz; + uint32_t sccc_buf_pg_sz; + uint32_t sccc_hop_num; + uint32_t qpc_timer_ba_pg_sz; + uint32_t qpc_timer_buf_pg_sz; + uint32_t qpc_timer_hop_num; + uint32_t cqc_timer_ba_pg_sz; + uint32_t cqc_timer_buf_pg_sz; + uint32_t cqc_timer_hop_num; + uint32_t cqe_ba_pg_sz; + uint32_t cqe_buf_pg_sz; + uint32_t cqe_hop_num; + uint32_t srqwqe_ba_pg_sz; + uint32_t srqwqe_buf_pg_sz; + uint32_t srqwqe_hop_num; + uint32_t idx_ba_pg_sz; + uint32_t idx_buf_pg_sz; + uint32_t idx_hop_num; + uint32_t eqe_ba_pg_sz; + uint32_t eqe_buf_pg_sz; + uint32_t eqe_hop_num; + uint32_t gmv_entry_num; + uint32_t gmv_entry_sz; + uint32_t gmv_ba_pg_sz; + uint32_t gmv_buf_pg_sz; + uint32_t gmv_hop_num; + uint32_t sl_num; + uint32_t llm_buf_pg_sz; + uint32_t llm_ba_idx; + uint32_t llm_ba_num; + uint32_t chunk_sz; /* chunk size in non multihop mode */ + uint64_t flags; + uint16_t default_ceq_max_cnt; + uint16_t default_ceq_period; + uint16_t default_aeq_max_cnt; + uint16_t default_aeq_period; + uint16_t default_aeq_arm_st; + uint16_t default_ceq_arm_st; + uint8_t cong_type; + uint8_t oor_en; + uint8_t reorder_cq_buffer_en; + uint8_t reorder_cap; + uint8_t reorder_cq_shift; + uint32_t onflight_size; + uint8_t dynamic_ack_timeout; + uint32_t num_jfc_shift; + uint32_t num_jfs_shift; + uint32_t num_jfr_shift; + uint32_t num_jetty_shift; + uint8_t poe_ch_num; + uint32_t speed; +}; + +struct udma_idx_table { + uint32_t *spare_idx; + uint32_t head; + uint32_t tail; +}; + +struct udma_bank { + struct ida ida; + uint32_t inuse; /* Number of IDs allocated */ + uint32_t min; /* Lowest ID to allocate. */ + uint32_t max; /* Highest ID to allocate. */ + uint32_t next; /* Next ID to allocate. */ +}; + +#define TRACE_AEQE_LEN_MAX 64 + +struct udma_eq { + struct udma_dev *udma_dev; + void __iomem *db_reg; + + int type_flag; /* Aeq:1 ceq:0 */ + int eqn; + uint32_t entries; + int eqe_size; + int irq; + uint32_t cons_index; + int over_ignore; + int coalesce; + int arm_st; + int hop_num; + struct udma_mtr mtr; + uint16_t eq_max_cnt; + uint32_t eq_period; + int shift; + int event_type; + int sub_type; +}; + +struct udma_qp_table { + struct xarray xa; + struct udma_hem_table qp_table; + struct udma_hem_table irrl_table; + struct udma_hem_table trrl_table; + struct udma_hem_table sccc_table; + struct udma_bank bank[UDMA_QP_BANK_NUM]; + struct mutex bank_mutex; + struct udma_idx_table idx_table; +}; + +struct udma_eq_table { + uint32_t *idx_table; + struct udma_eq *eq; +}; + +struct udma_jfc_table { + struct xarray xa; + struct udma_hem_table table; + struct udma_bank bank[UDMA_CQ_BANK_NUM]; + struct mutex bank_mutex; +}; + +struct udma_jfs_table { + struct xarray xa; + struct udma_ida jfs_ida; +}; + +struct udma_jfr_table { + struct xarray xa; + struct udma_hem_table table; + struct udma_ida jfr_ida; +}; + +struct udma_jetty_table { + struct xarray xa; + struct udma_ida jetty_ida; +}; + +struct udma_seg_table { + struct udma_ida seg_ida; + struct udma_hem_table table; +}; + +#define UDMA_SCC_PARAM_SIZE 4 + +struct udma_scc_param { + uint32_t param[UDMA_SCC_PARAM_SIZE]; + uint32_t lifespan; + uint64_t timestamp; + enum udma_cong_type algo_type; + struct delayed_work scc_cfg_dwork; + struct udma_dev *udma_dev; + uint8_t port_num; + bool configured; +}; + +struct udma_port { + struct udma_dev *udma_dev; + uint8_t port_num; + struct kobject kobj; + struct udma_scc_param *scc_param; +}; + +struct udma_dev { + struct ubcore_device ub_dev; + struct pci_dev *pci_dev; + struct device *dev; + + bool is_reset; + bool dis_db; + uint64_t reset_cnt; + struct udma_netdev uboe; + + struct list_head pgdir_list; + struct mutex pgdir_mutex; + uint8_t __iomem *reg_base; + struct udma_caps caps; + + int irq_num; + int irq[UDMA_MAX_IRQ_NUM]; + const char *irq_names[UDMA_MAX_IRQ_NUM]; + char dev_name[UBCORE_MAX_DEV_NAME]; + uint64_t sys_image_guid; + struct udma_cmdq cmd; + int cmd_mod; + struct page *reset_page; /* store reset state */ + void *reset_kaddr; /* addr of reset page */ + const struct udma_hw *hw; + void *priv; + struct workqueue_struct *irq_workq; + struct work_struct ecc_work; + uint16_t func_id; + uint32_t func_num; + uint32_t cong_algo_tmpl_id; + struct udma_ida uar_ida; + struct udma_jfs_table jfs_table; + struct udma_jfr_table jfr_table; + struct udma_jetty_table jetty_table; + struct udma_seg_table seg_table; + struct udma_jfc_table jfc_table; + struct udma_qp_table qp_table; + struct udma_eq_table eq_table; + struct udma_hem_table qpc_timer_table; + struct udma_hem_table cqc_timer_table; + struct udma_hem_table gmv_table; + uint64_t dwqe_page; + uint64_t dfx_cnt[UDMA_DFX_EQ_TOTAL]; + struct list_head qp_list; + spinlock_t qp_list_lock; + struct list_head dip_list; + spinlock_t dip_list_lock; + struct udma_port port_data[UDMA_MAX_PORTS]; +}; + +struct udma_seg { + struct ubcore_target_seg ubcore_seg; + uint64_t iova; + uint64_t size; + uint32_t key; + uint32_t pd; + uint32_t access; + int enabled; + uint32_t pbl_hop_num; + struct udma_mtr pbl_mtr; + uint32_t npages; +}; + +static inline void *udma_buf_offset(struct udma_buf *buf, + uint32_t offset) +{ + return (char *)(buf->trunk_list[offset >> buf->trunk_shift].buf) + + (offset & ((1 << buf->trunk_shift) - 1)); +} + +static inline uint64_t to_hr_hw_page_addr(uint64_t addr) +{ + return addr >> UDMA_HW_PAGE_SHIFT; +} + +static inline uint32_t to_hr_hw_page_shift(uint32_t page_shift) +{ + return page_shift - UDMA_HW_PAGE_SHIFT; +} + +static inline uint32_t to_udma_hem_hopnum(uint32_t hopnum, uint32_t count) +{ + if (count > 0) + return hopnum == UDMA_HOP_NUM_0 ? 0 : hopnum; + + return 0; +} + +static inline struct udma_ucontext + *to_udma_ucontext(struct ubcore_ucontext *uctx) +{ + return container_of(uctx, struct udma_ucontext, uctx); +} + +static inline struct udma_dev *to_udma_dev(const struct ubcore_device *ubcore_dev) +{ + return container_of(ubcore_dev, struct udma_dev, ub_dev); +} + +static inline struct udma_seg *to_udma_seg(struct ubcore_target_seg *seg) +{ + return container_of(seg, struct udma_seg, ubcore_seg); +} + +static inline uint32_t to_udma_hem_entries_size(uint32_t count, + uint32_t buf_shift) +{ + return udma_hw_page_align(count << buf_shift); +} + +static inline uint32_t to_udma_hw_page_shift(uint32_t page_shift) +{ + return page_shift - UDMA_HW_PAGE_SHIFT; +} + +static inline uint32_t to_udma_hem_entries_count(uint32_t count, + uint32_t buf_shift) +{ + return udma_hw_page_align(count << buf_shift) >> buf_shift; +} + +static inline uint32_t to_udma_hem_entries_shift(uint32_t count, + uint32_t buf_shift) +{ + if (!count) + return 0; + + return ilog2(to_udma_hem_entries_count(count, buf_shift)); +} + +static inline uint64_t to_udma_hw_page_addr(uint64_t addr) +{ + return addr >> UDMA_HW_PAGE_SHIFT; +} + +static inline dma_addr_t udma_buf_dma_addr(struct udma_buf *buf, + uint32_t offset) +{ + return buf->trunk_list[offset >> buf->trunk_shift].map + + (offset & ((1 << buf->trunk_shift) - 1)); +} + +static inline dma_addr_t udma_buf_page(struct udma_buf *buf, uint32_t idx) +{ + return udma_buf_dma_addr(buf, idx << buf->page_shift); +} + +int udma_cmd_init(struct udma_dev *udma_dev); +void udma_cmd_cleanup(struct udma_dev *udma_dev); +int udma_cmd_use_events(struct udma_dev *udma_dev); +void udma_cmd_use_polling(struct udma_dev *udma_dev); +int udma_cmq_send(struct udma_dev *udma_dev, + struct udma_cmq_desc *desc, int num); +int udma_hnae_client_init(struct udma_dev *udma_dev); +void udma_hnae_client_exit(struct udma_dev *udma_dev); +int udma_mtr_create(struct udma_dev *udma_dev, struct udma_mtr *mtr, + struct udma_buf_attr *buf_attr, uint32_t ba_page_shift, + uint64_t user_addr, bool is_user); +void udma_mtr_destroy(struct udma_dev *udma_dev, struct udma_mtr *mtr); +int udma_init_qp_table(struct udma_dev *udma_dev); +void udma_cleanup_qp_table(struct udma_dev *udma_dev); +void udma_cleanup_common_hem(struct udma_dev *udma_dev); +int udma_init_common_hem(struct udma_dev *udma_dev); +void udma_destroy_eqc(struct udma_dev *udma_dev, int eqn, uint32_t eq_cmd); + +#endif /* _UDMA_DEVICE_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_dfx.c b/drivers/ub/hw/hns3/hns3_udma_dfx.c new file mode 100644 index 0000000000000000000000000000000000000000..c7437290ecb0330787b818fbf8619209573b8398 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_dfx.c @@ -0,0 +1,1161 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include +#include "hns3_udma_jfr.h" +#include "hns3_udma_cmd.h" +#include "hns3_udma_tp.h" +#include "hns3_udma_jfc.h" +#include "hns3_udma_segment.h" +#include "hns3_udma_dfx.h" + +struct class *drv_class; +struct device *drv_device; +static int major; +static int udma_dev_count; +struct udma_dfx_dev g_udma_dfx_list[MAX_UDMA_DEV] = {NULL}; +const struct file_operations chr_ops = { + .owner = THIS_MODULE, +}; + +static int udma_dfx_read_buf(char *str, const char *buf) +{ + const char *str_buf = buf; + int blk_cnt = 0; + int cnt = 0; + + while (*str_buf == ' ') { + str_buf++; + blk_cnt++; + } + while (str_buf[cnt] != ' ' && str_buf[cnt] != '\0' && + cnt < UDMA_DFX_STR_LEN_MAX - 1) + cnt++; + + if (((uint32_t)(blk_cnt + cnt) < strlen(buf)) || str_buf[cnt] != '\0') + return -EINVAL; + + memcpy(str, str_buf, cnt); + str[cnt] = '\0'; + + return 0; +} + +static int udma_dfx_query_context(struct udma_dev *udma_dev, uint32_t id, + void *context, uint32_t len, uint16_t op) +{ + struct udma_cmd_mailbox *mailbox; + struct udma_cmq_desc desc; + struct udma_mbox *mb; + int ret; + + mailbox = udma_alloc_cmd_mailbox(udma_dev); + if (IS_ERR(mailbox)) { + dev_err(udma_dev->dev, "alloc mailbox failed\n"); + ret = PTR_ERR(mailbox); + goto alloc_mailbox_fail; + } + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mb = (struct udma_mbox *)desc.data; + mbox_desc_init(mb, 0, mailbox->dma, id, op); + + ret = udma_cmd_mbox(udma_dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); + if (ret) { + dev_err(udma_dev->dev, "QUERY id(0x%x) cmd(0x%x) error(%d).\n", + id, op, ret); + goto err_mailbox; + } + memcpy(context, mailbox->buf, len); + +err_mailbox: + udma_free_cmd_mailbox(udma_dev, mailbox); +alloc_mailbox_fail: + return ret; +} + +static void udma_dfx_seg_print(struct udma_dev *udma_dev, uint32_t seg_key, + struct udma_mpt_entry *mpt_entry) +{ + uint32_t *mpt = (uint32_t *)mpt_entry; + uint32_t i; + + dev_info(udma_dev->dev, + "************ SEG/MPT(0x%8x) ENTRY INFO *************\n", + seg_key); + for (i = 0; i < (sizeof(*mpt_entry) / sizeof(uint32_t)); i++) { + pr_info("MPT(byte%4lu): %08x\n", (i + 1) * sizeof(uint32_t), *mpt); + mpt++; + } + dev_info(udma_dev->dev, + "*********************************************************\n"); +} + +static int udma_dfx_seg_store(const char *p_buf, struct udma_dfx_info *udma_dfx) +{ + struct udma_dev *udma_dev = (struct udma_dev *)udma_dfx->priv; + struct udma_mpt_entry mpt_entry; + char str[UDMA_DFX_STR_LEN_MAX]; + uint32_t mpt_index; + uint32_t seg_key; + int ret; + + ret = udma_dfx_read_buf(str, p_buf); + if (ret) { + dev_info(udma_dev->dev, "the inputing is invalid\n"); + return ret; + } + + if (kstrtouint(str, 0, &seg_key)) { + dev_err(udma_dev->dev, "convert str failed\n"); + return -EINVAL; + } + + mpt_index = key_to_hw_index(seg_key) & (udma_dev->caps.num_mtpts - 1); + ret = udma_dfx_query_context(udma_dev, mpt_index, &mpt_entry, + sizeof(mpt_entry), UDMA_CMD_QUERY_MPT); + if (ret) { + dev_err(udma_dev->dev, "query seg context failed, ret = %d\n", + ret); + return ret; + } + + udma_dfx_seg_print(udma_dev, seg_key, &mpt_entry); + return ret; +} + +static void udma_dfx_qpc_print(struct udma_dev *udma_dev, uint32_t qpn, + struct udma_qp_context *qp_context) +{ + uint32_t *qpc = (uint32_t *)qp_context; + uint32_t i; + + dev_info(udma_dev->dev, + "************ TP/QP(0x%8x) CONTEXT INFO *************\n", + qpn); + for (i = 0; i < (sizeof(*qp_context) / sizeof(uint32_t)); i++) { + pr_info("QPC(byte%4lu): %08x\n", (i + 1) * sizeof(uint32_t), *qpc); + qpc++; + } + dev_info(udma_dev->dev, + "*********************************************************\n"); +} + +static int udma_dfx_tp_store(const char *p_buf, struct udma_dfx_info *udma_dfx) +{ + struct udma_dev *udma_dev = (struct udma_dev *)udma_dfx->priv; + struct udma_qp_context qp_context; + char str[UDMA_DFX_STR_LEN_MAX]; + uint32_t tpn; + int ret; + + ret = udma_dfx_read_buf(str, p_buf); + if (ret) { + dev_info(udma_dev->dev, "the inputing is invalid\n"); + return ret; + } + + if (kstrtouint(str, 0, &tpn)) { + dev_err(udma_dev->dev, "convert str failed\n"); + return -EINVAL; + } + + ret = udma_dfx_query_context(udma_dev, tpn, &qp_context, + sizeof(qp_context), UDMA_CMD_QUERY_QPC); + if (ret) { + dev_err(udma_dev->dev, + "query qp context failed, ret = %d\n", ret); + return ret; + } + + udma_dfx_qpc_print(udma_dev, tpn, &qp_context); + return 0; +} + +static void udma_dfx_jfrc_print(struct udma_dev *udma_dev, uint32_t jfrn, + struct udma_jfr_context *jfr_context) +{ + uint32_t *jfrc = (uint32_t *)jfr_context; + uint32_t i; + + dev_info(udma_dev->dev, + "************ JFR/SRQ(0x%8x) CONTEXT INFO *************\n", + jfrn); + for (i = 0; i < (sizeof(*jfr_context) / sizeof(uint32_t)); i++) { + pr_info("SRQC(byte%4lu): %08x\n", (i + 1) * sizeof(uint32_t), + *jfrc); + jfrc++; + } + dev_info(udma_dev->dev, + "*********************************************************\n"); +} + +static int udma_dfx_jfr_store(const char *p_buf, struct udma_dfx_info *udma_dfx) +{ + struct udma_dev *udma_dev = (struct udma_dev *)udma_dfx->priv; + struct udma_jfr_context jfr_context; + char str[UDMA_DFX_STR_LEN_MAX]; + uint32_t jfrn; + int ret; + + ret = udma_dfx_read_buf(str, p_buf); + if (ret) { + dev_info(udma_dev->dev, "the inputing is invalid\n"); + return ret; + } + + if (kstrtouint(str, 0, &jfrn)) { + dev_err(udma_dev->dev, "convert str failed\n"); + return -EINVAL; + } + + ret = udma_dfx_query_context(udma_dev, jfrn, &jfr_context, + sizeof(jfr_context), UDMA_CMD_QUERY_SRQC); + if (ret) { + dev_err(udma_dev->dev, + "query jfr context failed, ret = %d\n", ret); + return ret; + } + + udma_dfx_jfrc_print(udma_dev, jfrn, &jfr_context); + return 0; +} + +static void udma_dfx_jfcc_print(struct udma_dev *udma_dev, uint32_t jfcn, + struct udma_jfc_context *jfc_context) +{ + int *jfcc = (int *)jfc_context; + uint32_t i; + + dev_info(udma_dev->dev, + "************ JFC/CQC(0x%8x) CONTEXT INFO *************\n", + jfcn); + for (i = 0; i < (sizeof(*jfc_context) / sizeof(int)); i++) { + pr_info("CQC(byte%4lu): %08x\n", (i + 1) * sizeof(int), *jfcc); + jfcc++; + } + dev_info(udma_dev->dev, + "*********************************************************\n"); +} + +static int udma_dfx_jfc_store(const char *p_buf, struct udma_dfx_info *udma_dfx) +{ + struct udma_dev *udma_dev = (struct udma_dev *)udma_dfx->priv; + struct udma_jfc_context jfc_context = {}; + char str[UDMA_DFX_STR_LEN_MAX] = {}; + uint32_t jfcn; + int ret; + + ret = udma_dfx_read_buf(str, p_buf); + if (ret) { + dev_info(udma_dev->dev, "the inputing is invalid\n"); + return ret; + } + + if (kstrtouint(str, 0, &jfcn)) { + dev_err(udma_dev->dev, "convert str failed\n"); + return -EINVAL; + } + + ret = udma_dfx_query_context(udma_dev, jfcn, &jfc_context, + sizeof(jfc_context), UDMA_CMD_QUERY_CQC); + if (ret) { + dev_info(udma_dev->dev, + "query jfc context fail, ret = %d, jfcn = %u\n", + ret, jfcn); + return ret; + } + + udma_dfx_jfcc_print(udma_dev, jfcn, &jfc_context); + return 0; +} + +int udma_find_dfx_dev(struct udma_dev *udma_dev, int *num) +{ + int i; + + for (i = 0; i < MAX_UDMA_DEV; i++) { + if (g_udma_dfx_list[i].dev == udma_dev) { + *num = i; + return 0; + } + } + + dev_err(udma_dev->dev, "failed to find dfx device!\n"); + return -EINVAL; +} + +static int udma_query_res_tp(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_tp_val *tp = (struct ubcore_res_tp_val *)val->addr; + struct udma_qp_context qp_context; + int ret; + + if (val->len < sizeof(struct ubcore_res_tp_val)) { + dev_err(udma_dev->dev, + "Failed to check len, type: %u, val->len: %u.\n", + (uint32_t)key->type, val->len); + val->len = sizeof(struct ubcore_res_tp_val); + return -EINVAL; + } + + ret = udma_dfx_query_context(udma_dev, key->key, &qp_context, + sizeof(qp_context), UDMA_CMD_QUERY_QPC); + if (ret) { + dev_err(udma_dev->dev, + "query qp context failed, ret = %d\n", ret); + return ret; + } + + tp->tpn = key->key; + tp->psn = udma_reg_read(&qp_context, QPC_SQ_CUR_PSN); + tp->pri = udma_reg_read(&qp_context, QPC_SL); + tp->oor = udma_reg_read(&qp_context.ext, QPCEX_OOR_EN); + tp->state = udma_reg_read(&qp_context, QPC_QP_ST); + tp->data_udp_start = udma_reg_read(&qp_context.ext, QPCEX_DATA_UDP_SRCPORT_L) | + udma_reg_read(&qp_context.ext, QPCEX_DATA_UDP_SRCPORT_H) << + QPCEX_DATA_UDP_SRCPORT_H_SHIFT; + tp->ack_udp_start = udma_reg_read(&qp_context.ext, QPCEX_ACK_UDP_SRCPORT); + tp->udp_range = udma_reg_read(&qp_context.ext, QPCEX_UDP_SRCPORT_RANGE); + tp->spray_en = udma_reg_read(&qp_context.ext, QPCEX_AR_EN); + + val->len = sizeof(struct ubcore_res_tp_val); + + return 0; +} + +static int udma_query_res_jfs(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_jfs_val *jfs = (struct ubcore_res_jfs_val *)val->addr; + struct jfs_list *jfs_now; + int ret; + int i; + + if (val->len < sizeof(struct ubcore_res_jfs_val)) { + dev_err(udma_dev->dev, + "Failed to check len, type: %u, val->len: %u.\n", + (uint32_t)key->type, val->len); + val->len = sizeof(struct ubcore_res_jfs_val); + return -EINVAL; + } + + ret = udma_find_dfx_dev(udma_dev, &i); + if (ret) + return ret; + + list_for_each_entry(jfs_now, + &g_udma_dfx_list[i].dfx->jfs_list->node, node) { + if (jfs_now->jfs_id == key->key) { + jfs->jfs_id = jfs_now->jfs_id; + jfs->state = jfs_now->state; + jfs->depth = jfs_now->depth; + jfs->pri = jfs_now->pri; + jfs->jfc_id = jfs_now->jfc_id; + val->len = sizeof(struct ubcore_res_jfs_val); + return 0; + } + } + + dev_err(udma_dev->dev, "failed to find jfs!\n"); + return -EINVAL; +} + +static int udma_query_res_jfr(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_jfr_val *jfr = (struct ubcore_res_jfr_val *)val->addr; + struct udma_jfr_context jfr_context; + struct jfr_list *jfr_now; + int ret; + int i; + + if (val->len < sizeof(struct ubcore_res_jfr_val)) { + dev_err(udma_dev->dev, + "Failed to check len, type: %u, val->len: %u.\n", + (uint32_t)key->type, val->len); + val->len = sizeof(struct ubcore_res_jfr_val); + return -EINVAL; + } + + ret = udma_dfx_query_context(udma_dev, key->key, &jfr_context, + sizeof(jfr_context), UDMA_CMD_QUERY_SRQC); + if (ret) { + dev_err(udma_dev->dev, + "query jfr context failed, ret = %d\n", ret); + return ret; + } + + ret = udma_find_dfx_dev(udma_dev, &i); + if (ret) + return ret; + + list_for_each_entry(jfr_now, + &g_udma_dfx_list[i].dfx->jfr_list->node, node) { + if (jfr_now->jfr_id == key->key) { + jfr->jfr_id = key->key; + jfr->state = udma_reg_read(&jfr_context, SRQC_SRQ_ST); + jfr->depth = 1 << udma_reg_read(&jfr_context, SRQC_SHIFT); + jfr->jfc_id = jfr_now->jfc_id; + val->len = sizeof(struct ubcore_res_jfr_val); + return 0; + } + } + + dev_err(udma_dev->dev, "failed to find jfr!\n"); + return -EINVAL; +} + +static int udma_query_res_jetty(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_jetty_val *jetty = (struct ubcore_res_jetty_val *)val->addr; + struct jetty_list *jetty_now; + int ret; + int i; + + if (val->len < sizeof(struct ubcore_res_jetty_val)) { + dev_err(udma_dev->dev, + "Failed to check len, type: %u, val->len: %u.\n", + (uint32_t)key->type, val->len); + val->len = sizeof(struct ubcore_res_jetty_val); + return -EINVAL; + } + + ret = udma_find_dfx_dev(udma_dev, &i); + if (ret) + return ret; + + list_for_each_entry(jetty_now, + &g_udma_dfx_list[i].dfx->jetty_list->node, node) { + if (jetty_now->jetty_id == key->key) { + jetty->jetty_id = jetty_now->jetty_id; + jetty->state = jetty_now->state; + jetty->jfs_depth = jetty_now->jfs_depth; + jetty->jfr_depth = jetty_now->jfr_depth; + jetty->pri = jetty_now->pri; + jetty->jfr_id = jetty_now->jfr_id; + jetty->send_jfc_id = jetty_now->jfc_s_id; + jetty->recv_jfc_id = jetty_now->jfc_r_id; + val->len = sizeof(struct ubcore_res_jetty_val); + return 0; + } + } + + dev_err(udma_dev->dev, "failed to find jetty!\n"); + return -EINVAL; +} + +static int udma_query_res_jfc(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_jfc_val *jfc = (struct ubcore_res_jfc_val *)val->addr; + struct udma_jfc_context jfc_context; + int ret; + + if (val->len < sizeof(struct ubcore_res_jfc_val)) { + dev_err(udma_dev->dev, + "Failed to check len, type: %u, val->len: %u.\n", + (uint32_t)key->type, val->len); + val->len = sizeof(struct ubcore_res_jfc_val); + return -EINVAL; + } + + ret = udma_dfx_query_context(udma_dev, key->key, &jfc_context, + sizeof(jfc_context), UDMA_CMD_QUERY_CQC); + if (ret) { + dev_err(udma_dev->dev, + "query jfc context failed, ret = %d\n", ret); + return ret; + } + + jfc->jfc_id = udma_reg_read(&jfc_context, CQC_CQN); + jfc->state = udma_reg_read(&jfc_context, CQC_CQ_ST); + jfc->depth = 1 << udma_reg_read(&jfc_context, CQC_SHIFT); + + val->len = sizeof(struct ubcore_res_jfc_val); + + return 0; +} + +static int udma_query_res_seg(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_seg_val *seg = (struct ubcore_res_seg_val *)val->addr; + struct udma_mpt_entry mpt_entry; + uint32_t mpt_index; + int ret; + + if (val->len < sizeof(struct ubcore_res_seg_val)) { + dev_err(udma_dev->dev, + "Failed to check len, type: %u, val->len: %u.\n", + (uint32_t)key->type, val->len); + val->len = sizeof(struct ubcore_res_seg_val); + return -EINVAL; + } + + mpt_index = key_to_hw_index(key->key) & (udma_dev->caps.num_mtpts - 1); + ret = udma_dfx_query_context(udma_dev, mpt_index, &mpt_entry, + sizeof(mpt_entry), UDMA_CMD_QUERY_MPT); + if (ret) { + dev_err(udma_dev->dev, + "query seg context failed, ret = %d\n", ret); + return ret; + } + + seg->ubva.eid = udma_dev->ub_dev.attr.eid; + seg->ubva.uasid = udma_reg_read(&mpt_entry, MPT_PD); + seg->ubva.va = udma_reg_read(&mpt_entry, MPT_VA_L) | + udma_reg_read(&mpt_entry, MPT_VA_H) << + MPT_VA_H_SHIFT; + seg->len = udma_reg_read(&mpt_entry, MPT_LEN_L) | + udma_reg_read(&mpt_entry, MPT_LEN_H) << + MPT_LEN_H_SHIFT; + seg->key_id = udma_reg_read(&mpt_entry, MPT_LKEY); + + val->len = sizeof(struct ubcore_res_seg_val); + + return 0; +} + +static int udma_query_res_dev_tp(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val, int i) +{ + struct ubcore_res_dev_val *dev = (struct ubcore_res_dev_val *)val->addr; + uint32_t *tp_list_ptr = dev->tp_list; + struct tpn_list *tpn_now; + + dev->tp_cnt = 0; + if (!g_udma_dfx_list[i].dfx) { + dev_err(udma_dev->dev, "query res_dev_tp failed!\n"); + return -EINVAL; + } + + list_for_each_entry(tpn_now, + &g_udma_dfx_list[i].dfx->tpn_list->node, node) { + *tp_list_ptr = tpn_now->tpn; + tp_list_ptr++; + dev->tp_cnt++; + if (dev->tp_cnt == MAX_TP_CNT) + break; + } + + return 0; +} + +static int udma_query_res_dev_jfs(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val, int i) +{ + struct ubcore_res_dev_val *dev = (struct ubcore_res_dev_val *)val->addr; + uint32_t *jfs_list_ptr = dev->jfs_list; + struct jfs_list *jfs_now; + + dev->jfs_cnt = 0; + if (!g_udma_dfx_list[i].dfx) { + dev_err(udma_dev->dev, "query res_dev_jfs failed!\n"); + return -EINVAL; + } + + list_for_each_entry(jfs_now, + &g_udma_dfx_list[i].dfx->jfs_list->node, node) { + *jfs_list_ptr = jfs_now->jfs_id; + jfs_list_ptr++; + dev->jfs_cnt++; + if (dev->jfs_cnt == MAX_JFS_CNT) + break; + } + return 0; +} + +static int udma_query_res_dev_jfr(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val, int i) +{ + struct ubcore_res_dev_val *dev = (struct ubcore_res_dev_val *)val->addr; + uint32_t *jfr_list_ptr = dev->jfr_list; + struct jfr_list *jfr_now; + + dev->jfr_cnt = 0; + if (!g_udma_dfx_list[i].dfx) { + dev_err(udma_dev->dev, "query res_dev_jfr failed!\n"); + return -EINVAL; + } + + list_for_each_entry(jfr_now, + &g_udma_dfx_list[i].dfx->jfr_list->node, node) { + *jfr_list_ptr = jfr_now->jfr_id; + jfr_list_ptr++; + dev->jfr_cnt++; + if (dev->jfr_cnt == MAX_JFR_CNT) + break; + } + return 0; +} + +static int udma_query_res_dev_jetty(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val, int i) +{ + struct ubcore_res_dev_val *dev = (struct ubcore_res_dev_val *)val->addr; + uint32_t *jetty_list_ptr = dev->jetty_list; + struct jetty_list *jetty_now; + + dev->jetty_cnt = 0; + if (!g_udma_dfx_list[i].dfx) { + dev_err(udma_dev->dev, "query res_dev_jetty failed!\n"); + return -EINVAL; + } + + list_for_each_entry(jetty_now, + &g_udma_dfx_list[i].dfx->jetty_list->node, node) { + *jetty_list_ptr = jetty_now->jetty_id; + jetty_list_ptr++; + dev->jetty_cnt++; + if (dev->jetty_cnt == MAX_JETTY_CNT) + break; + } + return 0; +} + +static int udma_query_res_dev_jfc(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val, int i) +{ + struct ubcore_res_dev_val *dev = (struct ubcore_res_dev_val *)val->addr; + uint32_t *jfc_list_ptr = dev->jfc_list; + struct jfc_list *jfc_now; + + dev->jfc_cnt = 0; + if (!g_udma_dfx_list[i].dfx) { + dev_err(udma_dev->dev, "query res_dev_jfc failed!\n"); + return -EINVAL; + } + + list_for_each_entry(jfc_now, + &g_udma_dfx_list[i].dfx->jfc_list->node, node) { + *jfc_list_ptr = jfc_now->jfc_id; + jfc_list_ptr++; + dev->jfc_cnt++; + if (dev->jfc_cnt == MAX_JFC_CNT) + break; + } + return 0; +} + +static int udma_query_res_dev_seg(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val, int i) +{ + struct ubcore_res_dev_val *dev = (struct ubcore_res_dev_val *)val->addr; + struct ubcore_seg_info *seg_list_ptr = dev->seg_list; + struct seg_list *seg_now; + + dev->seg_cnt = 0; + if (!g_udma_dfx_list[i].dfx) { + dev_err(udma_dev->dev, "query res_dev_seg failed!\n"); + return -EINVAL; + } + + list_for_each_entry(seg_now, + &g_udma_dfx_list[i].dfx->seg_list->node, node) { + seg_list_ptr->ubva.eid = udma_dev->ub_dev.attr.eid; + seg_list_ptr->ubva.uasid = seg_now->pd; + seg_list_ptr->ubva.va = seg_now->iova; + seg_list_ptr->len = seg_now->len; + seg_list_ptr->key_id = seg_now->key_id; + seg_list_ptr++; + dev->seg_cnt++; + if (dev->seg_cnt == MAX_SEG_CNT) + break; + } + return 0; +} + +static int udma_query_res_dev(struct udma_dev *udma_dev, + struct ubcore_res_key *key, + struct ubcore_res_val *val) +{ + struct ubcore_res_dev_val *dev = (struct ubcore_res_dev_val *)val->addr; + int ret; + int i; + + if (val->len < sizeof(struct ubcore_res_dev_val)) { + dev_err(udma_dev->dev, + "Failed to check len, type: %u, val->len: %u.\n", + (uint32_t)key->type, val->len); + val->len = sizeof(struct ubcore_res_dev_val); + return -EINVAL; + } + + ret = udma_find_dfx_dev(udma_dev, &i); + if (ret) + return ret; + + ret = udma_query_res_dev_tp(udma_dev, key, val, i); + if (ret) + return ret; + + ret = udma_query_res_dev_jfs(udma_dev, key, val, i); + if (ret) + return ret; + + ret = udma_query_res_dev_jfr(udma_dev, key, val, i); + if (ret) + return ret; + + ret = udma_query_res_dev_jetty(udma_dev, key, val, i); + if (ret) + return ret; + + ret = udma_query_res_dev_jfc(udma_dev, key, val, i); + if (ret) + return ret; + + ret = udma_query_res_dev_seg(udma_dev, key, val, i); + if (ret) + return ret; + + dev->tpg_cnt = 0; + dev->utp_cnt = 0; + dev->jetty_group_cnt = 0; + + return 0; +} + +static int udma_check_key_type(struct udma_dev *udma_dev, + struct ubcore_res_key *key) +{ + bool ret; + + ret = key->type < UBCORE_RES_KEY_UPI || + key->type > UBCORE_RES_KEY_URMA_DEV || + key->type == UBCORE_RES_KEY_UPI || + key->type == UBCORE_RES_KEY_TPG || + key->type == UBCORE_RES_KEY_UTP || + key->type == UBCORE_RES_KEY_JETTY_GROUP; + if (ret) { + dev_err(udma_dev->dev, + "Key type: %u invalid.\n", (uint32_t)key->type); + return -EINVAL; + } + + return 0; +} + +int udma_query_res(const struct ubcore_device *dev, + struct ubcore_res_key *key, struct ubcore_res_val *val) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + int ret; + + ret = udma_check_key_type(udma_dev, key); + if (ret) + return ret; + + switch (key->type) { + case UBCORE_RES_KEY_TP: + return udma_query_res_tp(udma_dev, key, val); + case UBCORE_RES_KEY_JFS: + return udma_query_res_jfs(udma_dev, key, val); + case UBCORE_RES_KEY_JFR: + return udma_query_res_jfr(udma_dev, key, val); + case UBCORE_RES_KEY_JETTY: + return udma_query_res_jetty(udma_dev, key, val); + case UBCORE_RES_KEY_JFC: + return udma_query_res_jfc(udma_dev, key, val); + case UBCORE_RES_KEY_SEG: + return udma_query_res_seg(udma_dev, key, val); + case UBCORE_RES_KEY_URMA_DEV: + return udma_query_res_dev(udma_dev, key, val); + default: + return -EINVAL; + } + + return 0; +} + +UDMA_DFX_FILE_ATTR_DEF(tp_context, NULL, udma_dfx_tp_store); +UDMA_DFX_FILE_ATTR_DEF(jfr_context, NULL, udma_dfx_jfr_store); +UDMA_DFX_FILE_ATTR_DEF(seg_context, NULL, udma_dfx_seg_store); +UDMA_DFX_FILE_ATTR_DEF(jfc_context, NULL, udma_dfx_jfc_store); + +static struct attribute *udma_dfx_attrs_list[] = { + HW_ATTRS_LIST_MEMBER(tp_context), + HW_ATTRS_LIST_MEMBER(jfr_context), + HW_ATTRS_LIST_MEMBER(seg_context), + HW_ATTRS_LIST_MEMBER(jfc_context), + NULL +}; + +static ssize_t udma_dfx_show(struct kobject *kobj, struct attribute *attr, + char *buf) +{ + struct udma_dfx_sys_attr *p_udma_sys_attr = + container_of(attr, struct udma_dfx_sys_attr, attr); + struct udma_dfx_info *udma_dfx = container_of(kobj, + struct udma_dfx_info, + kobj); + int ret; + + memset(buf, 0, PAGE_SIZE); + if (p_udma_sys_attr->pub_show) { + ret = p_udma_sys_attr->pub_show(udma_dfx); + if (ret) + return ret; + else + return strlen(buf); + } + + return -EPERM; +} + +static ssize_t udma_dfx_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count) +{ + struct udma_dfx_sys_attr *p_udma_sys_attr = + container_of(attr, struct udma_dfx_sys_attr, attr); + struct udma_dfx_info *udma_dfx = container_of(kobj, + struct udma_dfx_info, + kobj); + int ret; + + if (p_udma_sys_attr->pub_store) { + ret = p_udma_sys_attr->pub_store((char *)buf, udma_dfx); + if (ret) + return ret; + else + return count; + } + + return -EPERM; +} + +static const struct sysfs_ops udma_dfx_file_ops = { + .show = udma_dfx_show, + .store = udma_dfx_store, +}; + +static struct kobj_type udma_dfx_kobj_ktype = { + .release = NULL, + .sysfs_ops = &udma_dfx_file_ops, + .default_attrs = udma_dfx_attrs_list, +}; + +static int udma_dfx_add_sysfs(struct udma_dfx_info *udma_dfx) +{ + struct device *dev = udma_dfx->drv_dev; + int ret; + + ret = kobject_init_and_add(&udma_dfx->kobj, + &udma_dfx_kobj_ktype, + &dev->kobj, + "%s", udma_dfx->dev.dev_name); + if (ret) + dev_err(drv_device, "kobject_init_and_add failed!\r\n"); + + return ret; +} + +static void udma_dfx_del_sysfs(struct udma_dfx_info *udma_dfx) +{ + kobject_del(&udma_dfx->kobj); +} + +struct udma_dfx_ops udma_dfx_ops = { + .add_sysfs = udma_dfx_add_sysfs, + .del_sysfs = udma_dfx_del_sysfs, +}; + +static void list_lock_init(struct udma_dfx_info *dfx) +{ + spin_lock_init(&dfx->tpn_list->node_lock); + INIT_LIST_HEAD(&dfx->tpn_list->node); + spin_lock_init(&dfx->jfs_list->node_lock); + INIT_LIST_HEAD(&dfx->jfs_list->node); + spin_lock_init(&dfx->jfr_list->node_lock); + INIT_LIST_HEAD(&dfx->jfr_list->node); + spin_lock_init(&dfx->jetty_list->node_lock); + INIT_LIST_HEAD(&dfx->jetty_list->node); + spin_lock_init(&dfx->jfc_list->node_lock); + INIT_LIST_HEAD(&dfx->jfc_list->node); + spin_lock_init(&dfx->seg_list->node_lock); + INIT_LIST_HEAD(&dfx->seg_list->node); +} + +static int udma_dfx_list_init(int num) +{ + struct udma_dfx_info *dfx; + int ret = -ENOMEM; + + dfx = g_udma_dfx_list[num].dfx; + + dfx->tpn_list = kzalloc(sizeof(struct tpn_list), GFP_KERNEL); + if (!dfx->tpn_list) + return ret; + + dfx->jfs_list = kzalloc(sizeof(struct jfs_list), GFP_KERNEL); + if (!dfx->jfs_list) + goto tpn_list_alloc_failed; + + dfx->jfr_list = kzalloc(sizeof(struct jfr_list), GFP_KERNEL); + if (!dfx->jfr_list) + goto jfs_id_list_alloc_failed; + + dfx->jetty_list = kzalloc(sizeof(struct jetty_list), GFP_KERNEL); + if (!dfx->jetty_list) + goto jfr_id_list_alloc_failed; + + dfx->jfc_list = kzalloc(sizeof(struct jfc_list), GFP_KERNEL); + if (!dfx->jfc_list) + goto jetty_id_list_alloc_failed; + + dfx->seg_list = kzalloc(sizeof(struct seg_list), GFP_KERNEL); + if (!dfx->seg_list) + goto jfc_id_list_alloc_failed; + + list_lock_init(dfx); + + return 0; + +jfc_id_list_alloc_failed: + kfree(dfx->jfc_list); +jetty_id_list_alloc_failed: + kfree(dfx->jetty_list); +jfr_id_list_alloc_failed: + kfree(dfx->jfr_list); +jfs_id_list_alloc_failed: + kfree(dfx->jfs_list); +tpn_list_alloc_failed: + kfree(dfx->tpn_list); + dev_err(drv_device, "dfx alloc list failed\n"); + + return ret; +} + +#define DFX_LIST_FREE(name) \ +do { \ + lock = &dfx->name##_list->node_lock; \ + spin_lock_irqsave(lock, flags); \ + list_for_each_entry_safe(name##_id, name##_tmp, \ + &dfx->name##_list->node, node) { \ + list_del(&name##_id->node); \ + kfree(name##_id); \ + } \ + spin_unlock_irqrestore(lock, flags); \ +} while (0) + +static void udma_dfx_list_free(int num) +{ + struct jetty_list *jetty_id, *jetty_tmp; + struct jfs_list *jfs_id, *jfs_tmp; + struct jfr_list *jfr_id, *jfr_tmp; + struct jfc_list *jfc_id, *jfc_tmp; + struct seg_list *seg_id, *seg_tmp; + struct tpn_list *tpn_id, *tpn_tmp; + struct udma_dfx_info *dfx; + unsigned long flags; + spinlock_t *lock; + + dfx = g_udma_dfx_list[num].dfx; + DFX_LIST_FREE(jetty); + DFX_LIST_FREE(tpn); + DFX_LIST_FREE(jfr); + DFX_LIST_FREE(jfs); + DFX_LIST_FREE(seg); + DFX_LIST_FREE(jfc); +} + +static int udma_dfx_add_udma_device(struct udma_dev *udma_dev) +{ + int ret; + int i; + + if (udma_dev_count == MAX_UDMA_DEV) { + dev_err(drv_device, + "udma dfx add device failed, g_udma_dfx_list is full\n."); + ret = -EINVAL; + goto g_udma_dfx_list_full; + } + for (i = 0; i < MAX_UDMA_DEV; i++) + if (!g_udma_dfx_list[i].dfx) + break; + + g_udma_dfx_list[i].dev = udma_dev; + g_udma_dfx_list[i].dfx = kzalloc(sizeof(struct udma_dfx_info), + GFP_KERNEL); + if (!g_udma_dfx_list[i].dfx) { + ret = -ENOMEM; + goto dfx_info_alloc_failed; + } + g_udma_dfx_list[i].dfx->priv = (void *)udma_dev; + g_udma_dfx_list[i].dfx->ops = &udma_dfx_ops; + g_udma_dfx_list[i].dfx->drv_dev = drv_device; + strlcpy(g_udma_dfx_list[i].dfx->dev.dev_name, udma_dev->dev_name, + UBCORE_MAX_DEV_NAME); + ret = udma_dfx_list_init(i); + if (ret) { + dev_err(drv_device, "dfx add dev list failed\n"); + goto dfx_info_alloc_failed; + } + + ret = g_udma_dfx_list[i].dfx->ops->add_sysfs(g_udma_dfx_list[i].dfx); + if (ret) { + dev_err(drv_device, "dfx add sysfs failed\n"); + goto add_sysfs_failed; + } + + dev_info(drv_device, "add udma device (%s) in udma dfx\n", + g_udma_dfx_list[i].dfx->dev.dev_name); + + udma_dev_count++; + + return 0; + +add_sysfs_failed: + udma_dfx_list_free(i); + kfree(g_udma_dfx_list[i].dfx); + g_udma_dfx_list[i].dfx = NULL; +dfx_info_alloc_failed: + g_udma_dfx_list[i].dev = NULL; +g_udma_dfx_list_full: + return ret; +} + +static int udma_dfx_chrdev_create(struct udma_dev *udma_dev) +{ + int ret; + + major = register_chrdev(0, DFX_DEVICE_NAME, &chr_ops); + if (major < 0) { + dev_err(udma_dev->dev, + "udma dfx register the character device failed\n "); + ret = major; + goto device_register_failed; + } + + drv_class = class_create(THIS_MODULE, DFX_DEVICE_NAME); + if (IS_ERR(drv_class)) { + dev_err(udma_dev->dev, "udma dfx class create failed\n"); + ret = (int)PTR_ERR(drv_class); + goto class_create_failed; + } + + drv_device = device_create(drv_class, NULL, MKDEV(major, 0), + NULL, DFX_DEVICE_NAME); + if (IS_ERR(drv_device)) { + dev_err(udma_dev->dev, "udma dfx create device failed\n"); + ret = (int)PTR_ERR(drv_device); + goto device_create_failed; + } + + return 0; + +device_create_failed: + drv_device = NULL; + class_destroy(drv_class); +class_create_failed: + drv_class = NULL; + unregister_chrdev(major, DFX_DEVICE_NAME); +device_register_failed: + major = -1; + return ret; +} + +static void udma_dfx_chrdev_destroy(void) +{ + device_destroy(drv_class, MKDEV(major, 0)); + drv_device = NULL; + class_destroy(drv_class); + drv_class = NULL; + unregister_chrdev(major, DFX_DEVICE_NAME); + major = -1; +} + +int udma_dfx_init(struct udma_dev *udma_dev) +{ + int ret; + + if (!udma_dev_count) { + ret = udma_dfx_chrdev_create(udma_dev); + if (ret) { + dev_err(drv_device, + "udma dfx create chr device failed\n"); + goto chrdev_create_failed; + } + dev_info(drv_device, "udma dfx create chr device success\n"); + } + + ret = udma_dfx_add_udma_device(udma_dev); + if (ret) { + dev_err(drv_device, "udma dfx add udma device failed\n"); + goto add_device_failed; + } + + return 0; + +add_device_failed: + if (!udma_dev_count) { + dev_info(drv_device, "udma dfx remove chr device\n"); + udma_dfx_chrdev_destroy(); + } +chrdev_create_failed: + return ret; +} + +static void udma_dfx_remove_udma_device(struct udma_dev *udma_dev) +{ + int i; + + for (i = 0; i < MAX_UDMA_DEV; i++) { + if (g_udma_dfx_list[i].dev) { + if (g_udma_dfx_list[i].dev == udma_dev) { + dev_info(drv_device, + "rmv udma device (%s) from udma dfx\n", + g_udma_dfx_list[i].dfx->dev.dev_name); + g_udma_dfx_list[i].dfx->ops->del_sysfs(g_udma_dfx_list[i].dfx); + udma_dfx_list_free(i); + + kfree(g_udma_dfx_list[i].dfx); + g_udma_dfx_list[i].dfx = NULL; + g_udma_dfx_list[i].dev = NULL; + udma_dev_count--; + break; + } + } + } +} + +void udma_dfx_uninit(struct udma_dev *udma_dev) +{ + if (!udma_dev_count) { + dev_err(udma_dev->dev, "no udma dfx device\n"); + return; + } + + udma_dfx_remove_udma_device(udma_dev); + if (!udma_dev_count) { + dev_info(drv_device, "udma dfx remove chr device\n"); + udma_dfx_chrdev_destroy(); + } +} diff --git a/drivers/ub/hw/hns3/hns3_udma_dfx.h b/drivers/ub/hw/hns3/hns3_udma_dfx.h new file mode 100644 index 0000000000000000000000000000000000000000..faacd0a536f940477de7362e2a98bc3ec7fef09e --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_dfx.h @@ -0,0 +1,139 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_DFX_H +#define _UDMA_DFX_H + +#include +#include "hns3_udma_device.h" + +#define DFX_DEVICE_NAME "udma_dfx" +#define MAX_UDMA_DEV 16 +#define MAX_TP_CNT 256 +#define MAX_JFS_CNT 256 +#define MAX_JFR_CNT 256 +#define MAX_JETTY_CNT 256 +#define MAX_JFC_CNT 256 +#define MAX_SEG_CNT 256 +#define UDMA_DFX_FILE_ATTR_DEF(file_name, func_show, func_store) \ +static struct udma_dfx_sys_attr g_sysfs_udma_##file_name##_attr = {\ + {\ + .name = #file_name,\ + .mode = 0644,\ + },\ + .pub_show = (func_show),\ + .pub_store = (func_store),\ +} + +#define HW_ATTRS_LIST_MEMBER(file_name) (&g_sysfs_udma_##file_name##_attr.attr) +#define MAX_CHAR_NUM_DEV_NAME 12 +#define UDMA_DFX_STR_LEN_MAX 20 + +struct udma_dfx_info; + +struct udma_dfx_sys_attr { + struct attribute attr; + int (*pub_show)(struct udma_dfx_info *udma_dfx); + int (*pub_store)(const char *buf, struct udma_dfx_info *udma_dfx); +}; + +struct udma_dfx_dev_info { + char dev_name[MAX_CHAR_NUM_DEV_NAME]; +}; + +struct udma_dfx_ops { + int (*add_sysfs)(struct udma_dfx_info *info); + void (*del_sysfs)(struct udma_dfx_info *info); +}; + +struct tpn_list { + uint32_t tpn; + struct list_head node; + spinlock_t node_lock; +}; + +struct jfs_list { + uint32_t jfs_id; + uint8_t state; + uint16_t depth; + uint8_t pri; + uint32_t jfc_id; + struct list_head node; + spinlock_t node_lock; +}; + +struct jfr_list { + uint32_t jfr_id; + uint32_t jfc_id; + struct list_head node; + spinlock_t node_lock; +}; + +struct jetty_list { + uint32_t jetty_id; + uint8_t state; + uint32_t jfs_depth; + uint32_t jfr_depth; + uint8_t pri; + uint32_t jfc_s_id; + uint32_t jfc_r_id; + uint32_t jfr_id; + struct list_head node; + spinlock_t node_lock; +}; + +struct jfc_list { + uint32_t jfc_id; + struct list_head node; + spinlock_t node_lock; +}; + +struct seg_list { + uint32_t pd; + uint64_t iova; + uint32_t len; + uint32_t key_id; + struct list_head node; + spinlock_t node_lock; +}; + +struct udma_dfx_info { + struct udma_dfx_dev_info dev; + struct udma_dfx_ops *ops; + struct device *drv_dev; + struct kobject kobj; + struct tpn_list *tpn_list; + struct jfs_list *jfs_list; + struct jfr_list *jfr_list; + struct jetty_list *jetty_list; + struct jfc_list *jfc_list; + struct seg_list *seg_list; + void *priv; +}; + +struct udma_dfx_dev { + struct udma_dfx_info *dfx; + struct udma_dev *dev; +}; + +extern struct udma_dfx_dev g_udma_dfx_list[MAX_UDMA_DEV]; + +int udma_dfx_init(struct udma_dev *udma_dev); +void udma_dfx_uninit(struct udma_dev *udma_dev); +int udma_find_dfx_dev(struct udma_dev *udma_dev, int *num); +int udma_query_res(const struct ubcore_device *dev, struct ubcore_res_key *key, + struct ubcore_res_val *val); + +#endif /* _UDMA_DFX_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_eq.c b/drivers/ub/hw/hns3/hns3_udma_eq.c new file mode 100644 index 0000000000000000000000000000000000000000..0710f130a463def3f149eace8cd2f6e3ba801d71 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_eq.c @@ -0,0 +1,902 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include +#include "hnae3.h" +#include "hns3_udma_hem.h" +#include "hns3_udma_device.h" +#include "hns3_udma_jfc.h" +#include "hns3_udma_jfr.h" +#include "hns3_udma_qp.h" +#include "hns3_udma_eq.h" + +static int alloc_eq_buf(struct udma_dev *udma_dev, struct udma_eq *eq) +{ + struct udma_buf_attr buf_attr = {}; + int err; + + if (udma_dev->caps.eqe_hop_num == UDMA_HOP_NUM_0) + eq->hop_num = 0; + else + eq->hop_num = udma_dev->caps.eqe_hop_num; + + buf_attr.page_shift = udma_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT; + buf_attr.region[0].size = eq->entries * eq->eqe_size; + buf_attr.region[0].hopnum = eq->hop_num; + buf_attr.region_count = 1; + + err = udma_mtr_create(udma_dev, &eq->mtr, &buf_attr, + udma_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT, 0, + 0); + if (err) + dev_err(udma_dev->dev, + "Failed to alloc EQE mtr, err %d\n", err); + + return err; +} + +static void init_eq_config(struct udma_dev *udma_dev, struct udma_eq *eq) +{ + eq->db_reg = udma_dev->reg_base + UDMA_VF_EQ_DB_CFG0_REG; + eq->cons_index = 0; + eq->over_ignore = UDMA_EQ_OVER_IGNORE_0; + eq->coalesce = UDMA_EQ_COALESCE_0; + eq->arm_st = UDMA_EQ_ALWAYS_ARMED; + eq->shift = ilog2((uint32_t)eq->entries); +} + +static int config_eqc(struct udma_dev *udma_dev, struct udma_eq *eq, + void *mb_buf) +{ + uint64_t eqe_ba[MTT_MIN_COUNT] = {}; + struct udma_eq_context *eqc; + uint64_t bt_ba = 0; + int count; + + eqc = (struct udma_eq_context *)mb_buf; + memset(eqc, 0, sizeof(struct udma_eq_context)); + + /* if not multi-hop, eqe buffer only use one trunk */ + count = udma_mtr_find(udma_dev, &eq->mtr, 0, eqe_ba, MTT_MIN_COUNT, + &bt_ba); + if (count < 1) { + dev_err(udma_dev->dev, "failed to find EQE mtr\n"); + return -ENOBUFS; + } + + udma_reg_write(eqc, EQC_EQ_ST, UDMA_EQ_STATE_VALID); + udma_reg_write(eqc, EQC_EQE_HOP_NUM, eq->hop_num); + udma_reg_write(eqc, EQC_OVER_IGNORE, eq->over_ignore); + udma_reg_write(eqc, EQC_COALESCE, eq->coalesce); + udma_reg_write(eqc, EQC_ARM_ST, eq->arm_st); + udma_reg_write(eqc, EQC_EQN, eq->eqn); + udma_reg_write(eqc, EQC_EQE_CNT, UDMA_EQ_INIT_EQE_CNT); + udma_reg_write(eqc, EQC_EQE_BA_PG_SZ, + to_udma_hw_page_shift(eq->mtr.hem_cfg.ba_pg_shift)); + udma_reg_write(eqc, EQC_EQE_BUF_PG_SZ, + to_udma_hw_page_shift(eq->mtr.hem_cfg.buf_pg_shift)); + udma_reg_write(eqc, EQC_EQ_PROD_INDX, UDMA_EQ_INIT_PROD_IDX); + udma_reg_write(eqc, EQC_EQ_MAX_CNT, eq->eq_max_cnt); + + udma_reg_write(eqc, EQC_EQ_PERIOD, eq->eq_period); + udma_reg_write(eqc, EQC_EQE_REPORT_TIMER, UDMA_EQ_INIT_REPORT_TIMER); + udma_reg_write(eqc, EQC_EQE_BA_L, bt_ba >> EQC_EQE_BA_L_SHIFT); + udma_reg_write(eqc, EQC_EQE_BA_H, bt_ba >> EQC_EQE_BA_H_SHIFT); + udma_reg_write(eqc, EQC_SHIFT, eq->shift); + udma_reg_write(eqc, EQC_MSI_INDX, UDMA_EQ_INIT_MSI_IDX); + udma_reg_write(eqc, EQC_CUR_EQE_BA_L, eqe_ba[0] >> + EQC_CUR_EQE_BA_L_SHIFT); + udma_reg_write(eqc, EQC_CUR_EQE_BA_M, eqe_ba[0] >> + EQC_CUR_EQE_BA_M_SHIFT); + udma_reg_write(eqc, EQC_CUR_EQE_BA_H, eqe_ba[0] >> + EQC_CUR_EQE_BA_H_SHIFT); + udma_reg_write(eqc, EQC_EQ_CONS_INDX, UDMA_EQ_INIT_CONS_IDX); + udma_reg_write(eqc, EQC_NEX_EQE_BA_L, eqe_ba[1] >> + EQC_NEX_EQE_BA_L_SHIFT); + udma_reg_write(eqc, EQC_NEX_EQE_BA_H, eqe_ba[1] >> + EQC_NEX_EQE_BA_H_SHIFT); + udma_reg_write(eqc, EQC_EQE_SIZE, eq->eqe_size == UDMA_EQE_SIZE); + + return 0; +} + +static void free_eq_buf(struct udma_dev *udma_dev, struct udma_eq *eq) +{ + udma_mtr_destroy(udma_dev, &eq->mtr); +} + +static int udma_create_eq(struct udma_dev *udma_dev, struct udma_eq *eq, + uint32_t eq_cmd) +{ + struct udma_cmd_mailbox *mailbox; + struct udma_cmq_desc desc; + struct udma_mbox *mb; + int ret; + + /* Allocate mailbox memory */ + mailbox = udma_alloc_cmd_mailbox(udma_dev); + if (IS_ERR_OR_NULL(mailbox)) + return -ENOMEM; + + ret = config_eqc(udma_dev, eq, mailbox->buf); + if (ret) + goto err_cmd_mbox; + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mbox_desc_init(mb, mailbox->dma, 0, eq->eqn, eq_cmd); + ret = udma_cmd_mbox(udma_dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); + if (ret) + dev_err(udma_dev->dev, "[mailbox cmd] create eqc failed.\n"); + +err_cmd_mbox: + udma_free_cmd_mailbox(udma_dev, mailbox); + + return ret; +} + +static struct udma_aeqe *next_aeqe_sw_v2(struct udma_eq *eq) +{ + struct udma_aeqe *aeqe; + + aeqe = (struct udma_aeqe *)udma_buf_offset(eq->mtr.kmem, + (eq->cons_index & (eq->entries - 1)) * + eq->eqe_size); + + return (udma_get_bit(aeqe->asyn, UDMA_AEQ_AEQE_OWNER_S) ^ + !!(eq->cons_index & eq->entries)) ? aeqe : NULL; +} + +static void aeq_event_dump(struct device *dev, struct udma_work *irq_work) +{ + switch (irq_work->event_type) { + case UDMA_EVENT_TYPE_COMM_EST: + break; + case UDMA_EVENT_TYPE_WQ_CATAS_ERROR: + dev_err(dev, "Local work queue 0x%x catast error, sub_event type is: 0x%x\n", + irq_work->queue_num, irq_work->sub_type); + break; + case UDMA_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: + dev_err(dev, + "Invalid request local work queue 0x%x error.\n", + irq_work->queue_num); + break; + case UDMA_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: + dev_err(dev, + "Local access violation work queue 0x%x error, sub_event type is: 0x%x\n", + irq_work->queue_num, irq_work->sub_type); + break; + case UDMA_EVENT_TYPE_JFR_LIMIT_REACH: + dev_warn(dev, "JFR limit reach.\n"); + break; + case UDMA_EVENT_TYPE_JFR_LAST_WQE_REACH: + dev_warn(dev, "JFR last wqe reach.\n"); + break; + case UDMA_EVENT_TYPE_JFC_ACCESS_ERROR: + dev_err(dev, "JFC 0x%x access err.\n", + irq_work->queue_num); + break; + case UDMA_EVENT_TYPE_JFC_OVERFLOW: + dev_warn(dev, "JFC 0x%x overflow\n", + irq_work->queue_num); + break; + default: + break; + } +} + +static void aeq_event_report(struct udma_dev *udma_dev, + struct udma_work *irq_work) +{ + uint32_t queue_num = irq_work->queue_num; + struct udma_aeqe *aeqe = &irq_work->aeqe; + int event_type = irq_work->event_type; + + switch (event_type) { + case UDMA_EVENT_TYPE_COMM_EST: + case UDMA_EVENT_TYPE_WQ_CATAS_ERROR: + case UDMA_EVENT_TYPE_JFR_LAST_WQE_REACH: + case UDMA_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: + case UDMA_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: + udma_qp_event(udma_dev, queue_num, event_type); + break; + case UDMA_EVENT_TYPE_JFR_LIMIT_REACH: + udma_jfr_event(udma_dev, queue_num, event_type); + break; + case UDMA_EVENT_TYPE_JFC_ACCESS_ERROR: + case UDMA_EVENT_TYPE_JFC_OVERFLOW: + udma_jfc_event(udma_dev, queue_num, event_type); + break; + case UDMA_EVENT_TYPE_MB: + udma_cmd_event(udma_dev, + le16_to_cpu(aeqe->event.cmd.token), + aeqe->event.cmd.status, + le64_to_cpu(aeqe->event.cmd.out_param)); + break; + default: + dev_err(udma_dev->dev, + "Unhandled event %d on EQ %d at idx %u.\n", + event_type, irq_work->eqn, irq_work->eq_ci); + break; + } +} + +static void udma_irq_work_handle(struct work_struct *work) +{ + struct udma_work *irq_work = + container_of(work, struct udma_work, work); + struct device *dev = irq_work->udma_dev->dev; + + aeq_event_dump(dev, irq_work); + aeq_event_report(irq_work->udma_dev, irq_work); + + kfree(irq_work); +} + +static void udma_init_irq_work(struct udma_dev *udma_dev, struct udma_eq *eq, + struct udma_aeqe *aeqe, uint32_t queue_num) +{ + struct udma_work *irq_work; + + irq_work = kzalloc(sizeof(struct udma_work), GFP_ATOMIC); + if (!irq_work) + return; + + irq_work->udma_dev = udma_dev; + irq_work->event_type = eq->event_type; + irq_work->sub_type = eq->sub_type; + irq_work->queue_num = queue_num; + irq_work->eq_ci = eq->cons_index; + irq_work->eqn = eq->eqn; + + memcpy(&irq_work->aeqe, aeqe, sizeof(struct udma_aeqe)); + + INIT_WORK(&irq_work->work, udma_irq_work_handle); + queue_work(udma_dev->irq_workq, &irq_work->work); +} + +static inline void udma_write64(struct udma_dev *udma_dev, uint32_t val[2], + void __iomem *dest) +{ + struct udma_priv *priv = (struct udma_priv *)udma_dev->priv; + struct hnae3_handle *handle = priv->handle; + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + + if (!udma_dev->dis_db && !ops->get_hw_reset_stat(handle)) + writeq(*(uint64_t *)val, dest); +} + +static void update_eq_db(struct udma_eq *eq) +{ + struct udma_dev *udma_dev = eq->udma_dev; + struct udma_eq_db eq_db = {}; + + if (eq->type_flag == UDMA_AEQ) { + udma_reg_write(&eq_db, UDMA_EQ_DB_CMD, + eq->arm_st == UDMA_EQ_ALWAYS_ARMED ? + UDMA_EQ_DB_CMD_AEQ : + UDMA_EQ_DB_CMD_AEQ_ARMED); + } else { + udma_reg_write(&eq_db, UDMA_EQ_DB_TAG, eq->eqn); + + udma_reg_write(&eq_db, UDMA_EQ_DB_CMD, + eq->arm_st == UDMA_EQ_ALWAYS_ARMED ? + UDMA_EQ_DB_CMD_CEQ : + UDMA_EQ_DB_CMD_CEQ_ARMED); + } + + udma_reg_write(&eq_db, UDMA_EQ_DB_CI, eq->cons_index); + + udma_write64(udma_dev, (uint32_t *)&eq_db, eq->db_reg); +} + +static int udma_aeq_int(struct udma_dev *udma_dev, struct udma_eq *eq) +{ + struct udma_aeqe *aeqe = next_aeqe_sw_v2(eq); + int aeqe_found = 0; + uint32_t queue_num; + int event_type; + uint32_t *tmp; + int sub_type; + + while (aeqe) { + /* Make sure we read AEQ entry after we have checked the + * ownership bit + */ + dma_rmb(); + + event_type = udma_get_field(aeqe->asyn, + UDMA_AEQE_EVENT_TYPE_M, + UDMA_AEQE_EVENT_TYPE_S); + sub_type = udma_get_field(aeqe->asyn, + UDMA_AEQE_SUB_TYPE_M, + UDMA_AEQE_SUB_TYPE_S); + queue_num = udma_get_field(aeqe->event.queue_event.num, + UDMA_AEQE_EVENT_QUEUE_NUM_M, + UDMA_AEQE_EVENT_QUEUE_NUM_S); + tmp = (uint32_t *)aeqe; + if (event_type != UDMA_EVENT_TYPE_COMM_EST) + dev_err(udma_dev->dev, "print AEQE: 0x%x, 0x%x; " + "queue:0x%x event_type:0x%x, sub_type:0x%x\n", + *tmp, *(tmp + 1), queue_num, event_type, + sub_type); + + eq->event_type = event_type; + eq->sub_type = sub_type; + aeqe_found = 1; + + ++eq->cons_index; + + udma_dev->dfx_cnt[UDMA_DFX_AEQE]++; + BUILD_BUG_ON(sizeof(struct udma_aeqe) > TRACE_AEQE_LEN_MAX); + + udma_init_irq_work(udma_dev, eq, aeqe, queue_num); + + aeqe = next_aeqe_sw_v2(eq); + } + + update_eq_db(eq); + return aeqe_found; +} + +static struct udma_ceqe *next_ceqe_sw_v2(struct udma_eq *eq) +{ + struct udma_ceqe *ceqe; + + ceqe = (struct udma_ceqe *)udma_buf_offset(eq->mtr.kmem, + (eq->cons_index & (eq->entries - 1)) * + eq->eqe_size); + + return (!!(udma_get_bit(ceqe->comp, UDMA_CEQ_CEQE_OWNER_S))) ^ + (!!(eq->cons_index & eq->entries)) ? ceqe : NULL; +} + +static int udma_ceq_int(struct udma_dev *udma_dev, + struct udma_eq *eq) +{ + struct udma_ceqe *ceqe = next_ceqe_sw_v2(eq); + int ceqe_found = 0; + uint32_t cqn; + + while (ceqe) { + /* Make sure we read CEQ entry after we have checked the + * ownership bit + */ + dma_rmb(); + + cqn = udma_get_field(ceqe->comp, UDMA_CEQE_COMP_CQN_M, + UDMA_CEQE_COMP_CQN_S); + + udma_jfc_completion(udma_dev, cqn); + + ++eq->cons_index; + udma_dev->dfx_cnt[UDMA_DFX_CEQE]++; + ceqe_found = 1; + + ceqe = next_ceqe_sw_v2(eq); + } + + update_eq_db(eq); + + return ceqe_found; +} + +static int fmea_ram_ecc_query(struct udma_dev *udma_dev, + struct fmea_ram_ecc *ecc_info) +{ + struct udma_cmq_desc desc; + struct udma_cmq_req *req = (struct udma_cmq_req *)desc.data; + int ret; + + udma_cmq_setup_basic_desc(&desc, UDMA_QUERY_RAM_ECC, true); + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) + return ret; + + ecc_info->is_ecc_err = udma_reg_read(req, QUERY_RAM_ECC_1BIT_ERR); + ecc_info->res_type = udma_reg_read(req, QUERY_RAM_ECC_RES_TYPE); + ecc_info->index = udma_reg_read(req, QUERY_RAM_ECC_TAG); + + return 0; +} + +static int fmea_recover_gmv(struct udma_dev *udma_dev, uint32_t idx) +{ + struct udma_cmq_desc desc; + struct udma_cmq_req *req = (struct udma_cmq_req *)desc.data; + uint32_t addr_upper; + uint32_t addr_low; + int ret; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_CFG_GMV_BT, true); + udma_reg_write(req, CFG_GMV_BT_IDX, idx); + udma_reg_write(req, CFG_GMV_BT_VF_ID, 0); + + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) { + dev_err(udma_dev->dev, + "failed to execute cmd to read gmv, ret = %d.\n", ret); + return ret; + } + + addr_low = udma_reg_read(req, CFG_GMV_BT_BA_L); + addr_upper = udma_reg_read(req, CFG_GMV_BT_BA_H); + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_CFG_GMV_BT, false); + udma_reg_write(req, CFG_GMV_BT_BA_L, addr_low); + udma_reg_write(req, CFG_GMV_BT_BA_H, addr_upper); + udma_reg_write(req, CFG_GMV_BT_IDX, idx); + udma_reg_write(req, CFG_GMV_BT_VF_ID, 0); + + return udma_cmq_send(udma_dev, &desc, 1); +} + +static uint64_t fmea_get_ram_res_addr(uint32_t res_type, uint64_t *data) +{ + if (res_type == ECC_RESOURCE_QPC_TIMER || + res_type == ECC_RESOURCE_CQC_TIMER || + res_type == ECC_RESOURCE_SCCC) + return le64_to_cpu(*data); + + return le64_to_cpu(*data) << PAGE_SHIFT; +} + +static int fmea_recover_others(struct udma_dev *udma_dev, uint32_t res_type, + uint32_t index) +{ + struct udma_cmd_mailbox *mailbox = udma_alloc_cmd_mailbox(udma_dev); + uint8_t write_bt0_op = fmea_ram_res[res_type].write_bt0_op; + uint8_t read_bt0_op = fmea_ram_res[res_type].read_bt0_op; + struct udma_cmq_desc desc; + struct udma_mbox *mb; + uint64_t addr; + int ret; + + if (IS_ERR_OR_NULL(mailbox)) + return PTR_ERR(mailbox); + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mbox_desc_init(mb, 0, mailbox->dma, index, read_bt0_op); + ret = udma_cmd_mbox(udma_dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); + if (ret) { + dev_err(udma_dev->dev, + "failed to execute cmd to read fmea ram, ret = %d.\n", + ret); + goto out; + } + + addr = fmea_get_ram_res_addr(res_type, (uint64_t *)(mailbox->buf)); + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mbox_desc_init(mb, addr, 0, index, write_bt0_op); + ret = udma_cmd_mbox(udma_dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); + if (ret) { + dev_err(udma_dev->dev, + "failed to execute cmd to write fmea ram, ret = %d.\n", + ret); + } + +out: + udma_free_cmd_mailbox(udma_dev, mailbox); + + return ret; +} + +static void fmea_ram_ecc_recover(struct udma_dev *udma_dev, + struct fmea_ram_ecc *ecc_info) +{ + uint32_t res_type = ecc_info->res_type; + uint32_t index = ecc_info->index; + int ret; + + BUILD_BUG_ON(ARRAY_SIZE(fmea_ram_res) != ECC_RESOURCE_COUNT); + + if (res_type >= ECC_RESOURCE_COUNT) { + dev_err(udma_dev->dev, "unsupported fmea ram ecc type %u.\n", + res_type); + return; + } + + if (res_type == ECC_RESOURCE_GMV) + ret = fmea_recover_gmv(udma_dev, index); + else + ret = fmea_recover_others(udma_dev, res_type, index); + + if (ret) + dev_err(udma_dev->dev, + "failed to recover %s, index = %u, ret = %d.\n", + fmea_ram_res[res_type].name, index, ret); +} + +static void fmea_ram_ecc_work(struct work_struct *ecc_work) +{ + struct udma_dev *udma_dev = + container_of(ecc_work, struct udma_dev, ecc_work); + struct fmea_ram_ecc ecc_info = {}; + + if (fmea_ram_ecc_query(udma_dev, &ecc_info)) { + dev_err(udma_dev->dev, "failed to query fmea ram ecc.\n"); + return; + } + + if (!ecc_info.is_ecc_err) { + dev_err(udma_dev->dev, "there is no fmea ram ecc found.\n"); + return; + } + + fmea_ram_ecc_recover(udma_dev, &ecc_info); +} + +static irqreturn_t abnormal_interrupt_basic(struct udma_dev *udma_dev, + uint32_t int_st) +{ + struct pci_dev *pdev = udma_dev->pci_dev; + struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev); + const struct hnae3_ae_ops *ops = ae_dev->ops; + irqreturn_t int_work = IRQ_NONE; + uint32_t int_en; + + int_en = ub_read(udma_dev, UDMA_VF_ABN_INT_EN_REG); + if (int_st & BIT(UDMA_VF_INT_ST_AEQ_OVERFLOW_S)) { + dev_err(udma_dev->dev, "AEQ overflow!\n"); + + ub_write(udma_dev, UDMA_VF_ABN_INT_ST_REG, + 1 << UDMA_VF_INT_ST_AEQ_OVERFLOW_S); + + /* Set reset level for reset_event() */ + if (ops->set_default_reset_request) + ops->set_default_reset_request(ae_dev, + HNAE3_FUNC_RESET); + if (ops->reset_event) + ops->reset_event(pdev, NULL); + + int_en |= 1 << UDMA_VF_ABN_INT_EN_S; + ub_write(udma_dev, UDMA_VF_ABN_INT_EN_REG, int_en); + + int_work = IRQ_HANDLED; + } else { + dev_err(udma_dev->dev, "there is no basic abn irq found.\n"); + } + + return IRQ_RETVAL(int_work); +} + +static irqreturn_t udma_msix_interrupt_abn(int irq, void *dev_id) +{ + struct udma_dev *udma_dev = (struct udma_dev *)dev_id; + int int_work = 0; + uint32_t int_st; + + int_st = ub_read(udma_dev, UDMA_VF_ABN_INT_ST_REG); + if (int_st) { + int_work = abnormal_interrupt_basic(udma_dev, int_st); + } else { + dev_err(udma_dev->dev, "ECC 1bit ERROR!\n"); + queue_work(udma_dev->irq_workq, &udma_dev->ecc_work); + int_work = IRQ_HANDLED; + } + + return IRQ_RETVAL(int_work); +} + +static irqreturn_t udma_msix_interrupt_eq(int irq, void *eq_ptr) +{ + struct udma_eq *eq = (struct udma_eq *)eq_ptr; + struct udma_dev *udma_dev = eq->udma_dev; + int int_work; + + if (eq->type_flag == UDMA_CEQ) + /* Completion event interrupt */ + int_work = udma_ceq_int(udma_dev, eq); + else + /* Asychronous event interrupt */ + int_work = udma_aeq_int(udma_dev, eq); + + return IRQ_RETVAL(int_work); +} + +static int alloc_and_set_irq_name(struct udma_dev *udma_dev, int irq_num, + int aeq_num, int other_num) +{ + int ret = 0; + int i; + + for (i = 0; i < irq_num; i++) { + udma_dev->irq_names[i] = kzalloc(UDMA_INT_NAME_LEN, GFP_KERNEL); + if (!udma_dev->irq_names[i]) { + ret = -ENOMEM; + goto err_kzalloc_failed; + } + } + + /* irq contains: abnormal + AEQ + CEQ */ + for (i = 0; i < other_num; i++) + snprintf((char *)udma_dev->irq_names[i], UDMA_INT_NAME_LEN, + "udma-abn-%d", i); + + for (i = other_num; i < (other_num + aeq_num); i++) + snprintf((char *)udma_dev->irq_names[i], UDMA_INT_NAME_LEN, + "udma-aeq-%d", i - other_num); + + for (i = (other_num + aeq_num); i < irq_num; i++) + snprintf((char *)udma_dev->irq_names[i], UDMA_INT_NAME_LEN, + "udma-ceq-%d", i - other_num - aeq_num); + return ret; + +err_kzalloc_failed: + for (i -= 1; i >= 0; i--) + kfree(udma_dev->irq_names[i]); + + return ret; +} + +static int udma_request_irq(struct udma_dev *udma_dev, int irq_num, + int ceq_num, int aeq_num, int other_num) +{ + struct udma_eq_table *eq_table = &udma_dev->eq_table; + int ret; + int j; + + ret = alloc_and_set_irq_name(udma_dev, irq_num, aeq_num, other_num); + if (ret) + return ret; + for (j = 0; j < irq_num; j++) { + if (j < other_num) + ret = request_irq(udma_dev->irq[j], + udma_msix_interrupt_abn, + 0, udma_dev->irq_names[j], udma_dev); + + else if (j < (other_num + ceq_num)) + ret = request_irq(eq_table->eq[j - other_num].irq, + udma_msix_interrupt_eq, + 0, udma_dev->irq_names[j + aeq_num], + &eq_table->eq[j - other_num]); + else + ret = request_irq(eq_table->eq[j - other_num].irq, + udma_msix_interrupt_eq, + 0, udma_dev->irq_names[j - ceq_num], + &eq_table->eq[j - other_num]); + if (ret) { + dev_err(udma_dev->dev, "Request irq error!\n"); + goto err_request_failed; + } + } + + return 0; + +err_request_failed: + for (j -= 1; j >= 0; j--) + if (j < other_num) + free_irq(udma_dev->irq[j], udma_dev); + else + free_irq(eq_table->eq[j - other_num].irq, + &eq_table->eq[j - other_num]); + for (j = irq_num - 1; j >= 0; j--) + kfree(udma_dev->irq_names[j]); + + return ret; +} + +static void udma_int_mask_enable(struct udma_dev *udma_dev, + int eq_num, uint32_t enable_flag) +{ + int i; + + for (i = 0; i < eq_num; i++) + ub_write(udma_dev, UDMA_VF_EVENT_INT_EN_REG + + i * EQ_REG_OFFSET, enable_flag); + + ub_write(udma_dev, UDMA_VF_ABN_INT_EN_REG, enable_flag); + ub_write(udma_dev, UDMA_VF_ABN_INT_CFG_REG, enable_flag); +} + +static void set_jfce_attr(struct udma_dev *udma_dev, struct udma_eq *eq, + int idx) +{ + eq->type_flag = UDMA_CEQ; + eq->entries = udma_dev->caps.ceqe_depth; + eq->eqe_size = udma_dev->caps.ceqe_size; + eq->irq = udma_dev->irq[idx]; + eq->eq_max_cnt = UDMA_CEQ_DEFAULT_BURST_NUM; + eq->eq_period = UDMA_CEQ_DEFAULT_INTERVAL; +} + +static void set_jfae_attr(struct udma_dev *udma_dev, struct udma_eq *eq, + int idx) +{ + eq->type_flag = UDMA_AEQ; + eq->entries = udma_dev->caps.aeqe_depth; + eq->eqe_size = udma_dev->caps.aeqe_size; + eq->irq = udma_dev->irq[idx]; + eq->eq_max_cnt = UDMA_AEQ_DEFAULT_BURST_NUM; + eq->eq_period = UDMA_AEQ_DEFAULT_INTERVAL; +} + +static int udma_create_hw_eq(struct udma_dev *udma_dev, int ceq_num, + int aeq_num, int other_num) +{ + struct udma_eq_table *eq_table = &udma_dev->eq_table; + struct udma_eq *eq; + uint32_t eq_cmd; + int eq_num; + int i; + int ret; + + eq_num = ceq_num + aeq_num; + + for (i = 0; i < eq_num; i++) { + eq = &eq_table->eq[i]; + eq->udma_dev = udma_dev; + eq->eqn = i; + if (i < ceq_num) { + /* JFCE */ + eq_cmd = UDMA_CMD_CREATE_CEQC; + set_jfce_attr(udma_dev, eq, (i + other_num + aeq_num)); + } else { + /* JFAE */ + eq_cmd = UDMA_CMD_CREATE_AEQC; + set_jfae_attr(udma_dev, eq, (i - ceq_num + other_num)); + } + init_eq_config(udma_dev, eq); + ret = alloc_eq_buf(udma_dev, eq); + if (ret) { + dev_err(udma_dev->dev, "failed to alloc eq buf.\n"); + goto err_out; + } + + ret = udma_create_eq(udma_dev, eq, eq_cmd); + if (ret) { + dev_err(udma_dev->dev, "failed to create eq.\n"); + free_eq_buf(udma_dev, &eq_table->eq[i]); + goto err_out; + } + } + return ret; + +err_out: + for (i -= 1; i >= 0; i--) + free_eq_buf(udma_dev, &eq_table->eq[i]); + + return ret; +} + +void udma_destroy_eqc(struct udma_dev *udma_dev, int eqn, uint32_t eq_cmd) +{ + struct device *dev = udma_dev->dev; + struct udma_cmq_desc desc; + struct udma_mbox *mb; + int ret; + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + + mbox_desc_init(mb, 0, 0, eqn & UDMA_EQN_M, eq_cmd); + + ret = udma_cmd_mbox(udma_dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); + if (ret) + dev_err(dev, "[mailbox cmd] destroy eqc(%d) failed.\n", eqn); +} + +static void __udma_free_irq(struct udma_dev *udma_dev) +{ + int irq_num; + int eq_num; + int i; + + eq_num = udma_dev->caps.num_comp_vectors + + udma_dev->caps.num_aeq_vectors; + irq_num = eq_num + udma_dev->caps.num_other_vectors; + + for (i = 0; i < udma_dev->caps.num_other_vectors; i++) + free_irq(udma_dev->irq[i], udma_dev); + + for (i = 0; i < eq_num; i++) + free_irq(udma_dev->eq_table.eq[i].irq, + &udma_dev->eq_table.eq[i]); + + for (i = 0; i < irq_num; i++) + kfree(udma_dev->irq_names[i]); +} + +void udma_cleanup_eq_table(struct udma_dev *udma_dev) +{ + struct udma_eq_table *eq_table = &udma_dev->eq_table; + uint32_t eq_cmd; + int eq_num; + int i; + + eq_num = udma_dev->caps.num_comp_vectors + + udma_dev->caps.num_aeq_vectors; + + /* Disable irq */ + udma_int_mask_enable(udma_dev, eq_num, EQ_DISABLE); + + __udma_free_irq(udma_dev); + flush_workqueue(udma_dev->irq_workq); + destroy_workqueue(udma_dev->irq_workq); + + for (i = 0; i < eq_num; i++) { + if (i < udma_dev->caps.num_comp_vectors) + eq_cmd = UDMA_CMD_DESTROY_CEQC; + else + eq_cmd = UDMA_CMD_DESTROY_AEQC; + + udma_destroy_eqc(udma_dev, i, eq_cmd); + free_eq_buf(udma_dev, &eq_table->eq[i]); + } + + kfree(eq_table->eq); +} + +int udma_init_eq_table(struct udma_dev *udma_dev) +{ + struct udma_eq_table *eq_table = &udma_dev->eq_table; + struct device *dev = udma_dev->dev; + int other_num; + int irq_num; + int ceq_num; + int aeq_num; + int eq_num; + int ret; + int i; + + other_num = udma_dev->caps.num_other_vectors; + ceq_num = udma_dev->caps.num_comp_vectors; + aeq_num = udma_dev->caps.num_aeq_vectors; + + eq_num = ceq_num + aeq_num; + irq_num = eq_num + other_num; + + eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL); + if (!eq_table->eq) + return -ENOMEM; + ret = udma_create_hw_eq(udma_dev, ceq_num, aeq_num, other_num); + if (ret) + goto err_create_eq_fail; + + INIT_WORK(&udma_dev->ecc_work, fmea_ram_ecc_work); + + udma_dev->irq_workq = alloc_ordered_workqueue("udma_irq_workq", 0); + if (!udma_dev->irq_workq) { + dev_err(dev, "failed to create irq workqueue.\n"); + ret = -ENOMEM; + goto err_alloc_workqueue_fail; + } + + ret = udma_request_irq(udma_dev, irq_num, ceq_num, aeq_num, + other_num); + if (ret) { + dev_err(dev, "failed to request irq.\n"); + goto err_request_irq_fail; + } + + /* enable irq */ + udma_int_mask_enable(udma_dev, eq_num, EQ_ENABLE); + + return 0; + +err_request_irq_fail: + destroy_workqueue(udma_dev->irq_workq); + +err_alloc_workqueue_fail: + for (i = ceq_num + aeq_num - 1; i >= 0; i--) + free_eq_buf(udma_dev, &eq_table->eq[i]); + +err_create_eq_fail: + kfree(eq_table->eq); + return ret; +} diff --git a/drivers/ub/hw/hns3/hns3_udma_eq.h b/drivers/ub/hw/hns3/hns3_udma_eq.h new file mode 100644 index 0000000000000000000000000000000000000000..bafd446fab3482816df6bec01265fe811e66ba85 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_eq.h @@ -0,0 +1,146 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_EQ_H +#define _UDMA_EQ_H + +#include +#include "hns3_udma_cmd.h" + +struct udma_eq_context { + uint32_t data[16]; +}; + +#define UDMA_EQ_STATE_VALID 1 +#define UDMA_EQ_INIT_EQE_CNT 0 +#define UDMA_EQ_INIT_PROD_IDX 0 +#define UDMA_EQ_INIT_REPORT_TIMER 0 +#define UDMA_EQ_INIT_MSI_IDX 0 +#define UDMA_EQ_INIT_CONS_IDX 0 +#define UDMA_EQ_INIT_NXT_EQE_BA 0 + +#define EQC_FIELD_LOC(h, l) ((uint64_t)(h) << 32 | (l)) + +#define EQC_EQ_ST EQC_FIELD_LOC(1, 0) +#define EQC_EQE_HOP_NUM EQC_FIELD_LOC(3, 2) +#define EQC_OVER_IGNORE EQC_FIELD_LOC(4, 4) +#define EQC_COALESCE EQC_FIELD_LOC(5, 5) +#define EQC_ARM_ST EQC_FIELD_LOC(7, 6) +#define EQC_EQN EQC_FIELD_LOC(15, 8) +#define EQC_EQE_CNT EQC_FIELD_LOC(31, 16) +#define EQC_EQE_BA_PG_SZ EQC_FIELD_LOC(35, 32) +#define EQC_EQE_BUF_PG_SZ EQC_FIELD_LOC(39, 36) +#define EQC_EQ_PROD_INDX EQC_FIELD_LOC(63, 40) +#define EQC_EQ_MAX_CNT EQC_FIELD_LOC(79, 64) +#define EQC_EQ_PERIOD EQC_FIELD_LOC(95, 80) +#define EQC_EQE_REPORT_TIMER EQC_FIELD_LOC(127, 96) +#define EQC_EQE_BA_L EQC_FIELD_LOC(159, 128) +#define EQC_EQE_BA_H EQC_FIELD_LOC(188, 160) +#define EQC_SHIFT EQC_FIELD_LOC(199, 192) +#define EQC_MSI_INDX EQC_FIELD_LOC(207, 200) +#define EQC_CUR_EQE_BA_L EQC_FIELD_LOC(223, 208) +#define EQC_CUR_EQE_BA_M EQC_FIELD_LOC(255, 224) +#define EQC_CUR_EQE_BA_H EQC_FIELD_LOC(259, 256) +#define EQC_EQ_CONS_INDX EQC_FIELD_LOC(287, 264) +#define EQC_NEX_EQE_BA_L EQC_FIELD_LOC(319, 288) +#define EQC_NEX_EQE_BA_H EQC_FIELD_LOC(339, 320) +#define EQC_EQE_SIZE EQC_FIELD_LOC(341, 340) + +#define UDMA_CEQE_COMP_CQN_S 0 +#define UDMA_CEQE_COMP_CQN_M GENMASK(23, 0) + +#define UDMA_AEQE_EVENT_TYPE_S 0 +#define UDMA_AEQE_EVENT_TYPE_M GENMASK(7, 0) + +#define UDMA_AEQE_SUB_TYPE_S 8 +#define UDMA_AEQE_SUB_TYPE_M GENMASK(15, 8) + +#define UDMA_AEQE_EVENT_QUEUE_NUM_S 0 +#define UDMA_AEQE_EVENT_QUEUE_NUM_M GENMASK(23, 0) + +#define EQC_EQE_BA_L_SHIFT 3 +#define EQC_EQE_BA_H_SHIFT 35 +#define EQC_CUR_EQE_BA_L_SHIFT 12 +#define EQC_CUR_EQE_BA_M_SHIFT 28 +#define EQC_CUR_EQE_BA_H_SHIFT 60 +#define EQC_NEX_EQE_BA_L_SHIFT 12 +#define EQC_NEX_EQE_BA_H_SHIFT 44 +#define UDMA_VF_INT_ST_AEQ_OVERFLOW_S 0 +#define UDMA_VF_ABN_INT_EN_S 0 + +struct udma_eq_db { + uint32_t data[2]; +}; + +struct fmea_ram_ecc { + uint32_t is_ecc_err; + uint32_t res_type; + uint32_t index; +}; + +enum ecc_resource_type { + ECC_RESOURCE_QPC, + ECC_RESOURCE_CQC, + ECC_RESOURCE_MPT, + ECC_RESOURCE_SRQC, + ECC_RESOURCE_GMV, + ECC_RESOURCE_QPC_TIMER, + ECC_RESOURCE_CQC_TIMER, + ECC_RESOURCE_SCCC, + ECC_RESOURCE_COUNT, +}; + +static const struct { + const char *name; + uint8_t read_bt0_op; + uint8_t write_bt0_op; +} fmea_ram_res[] = { + { "ECC_RESOURCE_QPC", + UDMA_CMD_READ_QPC_BT0, UDMA_CMD_WRITE_QPC_BT0 }, + { "ECC_RESOURCE_CQC", + UDMA_CMD_READ_CQC_BT0, UDMA_CMD_WRITE_CQC_BT0 }, + { "ECC_RESOURCE_MPT", + UDMA_CMD_READ_MPT_BT0, UDMA_CMD_WRITE_MPT_BT0 }, + { "ECC_RESOURCE_SRQC", + UDMA_CMD_READ_SRQC_BT0, UDMA_CMD_WRITE_SRQC_BT0 }, + /* ECC_RESOURCE_GMV is handled by cmdq, not mailbox */ + { "ECC_RESOURCE_GMV", + 0, 0 }, + { "ECC_RESOURCE_QPC_TIMER", + UDMA_CMD_READ_QPC_TIMER_BT0, UDMA_CMD_WRITE_QPC_TIMER_BT0 }, + { "ECC_RESOURCE_CQC_TIMER", + UDMA_CMD_READ_CQC_TIMER_BT0, UDMA_CMD_WRITE_CQC_TIMER_BT0 }, + { "ECC_RESOURCE_SCCC", + UDMA_CMD_READ_SCCC_BT0, UDMA_CMD_WRITE_SCCC_BT0 }, +}; + +#define EQ_DB_FIELD_LOC(h, l) ((uint64_t)(h) << 32 | (l)) + +#define UDMA_EQ_DB_TAG EQ_DB_FIELD_LOC(7, 0) +#define UDMA_EQ_DB_CMD EQ_DB_FIELD_LOC(17, 16) +#define UDMA_EQ_DB_CI EQ_DB_FIELD_LOC(55, 32) + +#define UDMA_EQ_ARMED 1 +#define UDMA_EQ_ALWAYS_ARMED 3 + +#define UDMA_EQ_DB_CMD_AEQ 0x0 +#define UDMA_EQ_DB_CMD_AEQ_ARMED 0x1 +#define UDMA_EQ_DB_CMD_CEQ 0x2 +#define UDMA_EQ_DB_CMD_CEQ_ARMED 0x3 + +int udma_init_eq_table(struct udma_dev *udma_dev); +void udma_cleanup_eq_table(struct udma_dev *udma_dev); + +#endif /* _UDMA_EQ_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_hem.c b/drivers/ub/hw/hns3/hns3_udma_hem.c new file mode 100644 index 0000000000000000000000000000000000000000..e425efaa258f3695d601a5e498baa964e7bb25df --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_hem.c @@ -0,0 +1,2031 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include "urma/ubcore_api.h" +#include "hns3_udma_device.h" +#include "hns3_udma_hem.h" + +bool udma_check_whether_mhop(struct udma_dev *udma_dev, uint32_t type) +{ + int hop_num = 0; + + switch (type) { + case HEM_TYPE_QPC: + hop_num = udma_dev->caps.qpc_hop_num; + break; + case HEM_TYPE_MTPT: + hop_num = udma_dev->caps.mpt_hop_num; + break; + case HEM_TYPE_CQC: + hop_num = udma_dev->caps.cqc_hop_num; + break; + case HEM_TYPE_SRQC: + hop_num = udma_dev->caps.srqc_hop_num; + break; + case HEM_TYPE_SCCC: + hop_num = udma_dev->caps.sccc_hop_num; + break; + case HEM_TYPE_QPC_TIMER: + hop_num = udma_dev->caps.qpc_timer_hop_num; + break; + case HEM_TYPE_CQC_TIMER: + hop_num = udma_dev->caps.cqc_timer_hop_num; + break; + case HEM_TYPE_GMV: + hop_num = udma_dev->caps.gmv_hop_num; + break; + default: + return false; + } + + return hop_num ? true : false; +} + +static bool udma_check_hem_null(struct udma_hem **hem, uint64_t hem_idx, + uint32_t bt_chunk_num, uint64_t hem_max_num) +{ + uint64_t start_idx = round_down(hem_idx, bt_chunk_num); + uint64_t check_max_num = start_idx + bt_chunk_num; + uint64_t i; + + for (i = start_idx; (i < check_max_num) && (i < hem_max_num); i++) + if (i != hem_idx && hem[i]) + return false; + + return true; +} + +static bool udma_check_bt_null(uint64_t **bt, uint64_t ba_idx, + uint32_t bt_chunk_num) +{ + uint64_t start_idx = round_down(ba_idx, bt_chunk_num); + uint32_t i; + + for (i = start_idx; i < bt_chunk_num; i++) + if (i != ba_idx && bt[i]) + return false; + + return true; +} + +static int udma_get_bt_num(uint32_t table_type, uint32_t hop_num) +{ + if (check_whether_bt_num_3(table_type, hop_num)) + return 3; + else if (check_whether_bt_num_2(table_type, hop_num)) + return 2; + else if (check_whether_bt_num_1(table_type, hop_num)) + return 1; + else + return 0; +} + +static int get_hem_table_config(struct udma_dev *udma_dev, + struct udma_hem_mhop *mhop, + uint32_t type) +{ + struct device *dev = udma_dev->dev; + + switch (type) { + case HEM_TYPE_QPC: + mhop->buf_chunk_size = 1 << (udma_dev->caps.qpc_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (udma_dev->caps.qpc_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = udma_dev->caps.qpc_bt_num; + mhop->hop_num = udma_dev->caps.qpc_hop_num; + break; + case HEM_TYPE_MTPT: + mhop->buf_chunk_size = 1 << (udma_dev->caps.mpt_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (udma_dev->caps.mpt_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = udma_dev->caps.mpt_bt_num; + mhop->hop_num = udma_dev->caps.mpt_hop_num; + break; + case HEM_TYPE_CQC: + mhop->buf_chunk_size = 1 << (udma_dev->caps.cqc_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (udma_dev->caps.cqc_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = udma_dev->caps.cqc_bt_num; + mhop->hop_num = udma_dev->caps.cqc_hop_num; + break; + case HEM_TYPE_SCCC: + mhop->buf_chunk_size = 1 << (udma_dev->caps.sccc_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (udma_dev->caps.sccc_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = udma_dev->caps.sccc_bt_num; + mhop->hop_num = udma_dev->caps.sccc_hop_num; + break; + case HEM_TYPE_QPC_TIMER: + mhop->buf_chunk_size = 1 << (udma_dev->caps.qpc_timer_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (udma_dev->caps.qpc_timer_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = udma_dev->caps.qpc_timer_bt_num; + mhop->hop_num = udma_dev->caps.qpc_timer_hop_num; + break; + case HEM_TYPE_CQC_TIMER: + mhop->buf_chunk_size = 1 << (udma_dev->caps.cqc_timer_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (udma_dev->caps.cqc_timer_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = udma_dev->caps.cqc_timer_bt_num; + mhop->hop_num = udma_dev->caps.cqc_timer_hop_num; + break; + case HEM_TYPE_SRQC: + mhop->buf_chunk_size = 1 << (udma_dev->caps.srqc_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (udma_dev->caps.srqc_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = udma_dev->caps.srqc_bt_num; + mhop->hop_num = udma_dev->caps.srqc_hop_num; + break; + case HEM_TYPE_GMV: + mhop->buf_chunk_size = 1 << (udma_dev->caps.gmv_buf_pg_sz + + PAGE_SHIFT); + mhop->bt_chunk_size = 1 << (udma_dev->caps.gmv_ba_pg_sz + + PAGE_SHIFT); + mhop->ba_l0_num = udma_dev->caps.gmv_bt_num; + mhop->hop_num = udma_dev->caps.gmv_hop_num; + break; + default: + dev_err(dev, "table %u not support multi-hop addressing!\n", + type); + return -EINVAL; + } + + return 0; +} + +int udma_calc_hem_mhop(struct udma_dev *udma_dev, + struct udma_hem_table *table, uint64_t *obj, + struct udma_hem_mhop *mhop) +{ + struct device *dev = udma_dev->dev; + uint32_t chunk_ba_num; + uint32_t chunk_size; + uint32_t table_idx; + uint32_t bt_num; + int ret; + + ret = get_hem_table_config(udma_dev, mhop, table->type); + if (ret) + return ret; + + if (!obj) + return 0; + + /* + * QPC/MTPT/CQC/SRQC/SCCC alloc hem for buffer pages. + * MTT/CQE alloc hem for bt pages. + */ + bt_num = udma_get_bt_num(table->type, mhop->hop_num); + chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; + chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : + mhop->bt_chunk_size; + table_idx = *obj / (chunk_size / table->obj_size); + switch (bt_num) { + case 3: + mhop->l2_idx = table_idx & (chunk_ba_num - 1); + mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1); + mhop->l0_idx = (table_idx / chunk_ba_num) / chunk_ba_num; + break; + case 2: + mhop->l1_idx = table_idx & (chunk_ba_num - 1); + mhop->l0_idx = table_idx / chunk_ba_num; + break; + case 1: + mhop->l0_idx = table_idx; + break; + default: + dev_err(dev, "table %u not support hop_num = %u!\n", + table->type, mhop->hop_num); + return -EINVAL; + } + if (mhop->l0_idx >= mhop->ba_l0_num) + mhop->l0_idx %= mhop->ba_l0_num; + + return 0; +} + +static void udma_free_hem(struct udma_dev *udma_dev, struct udma_hem *hem) +{ + struct udma_hem_chunk *chunk, *tmp; + int i; + + if (!hem) + return; + + list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) { + for (i = 0; i < chunk->npages; ++i) + dma_free_coherent(udma_dev->dev, + sg_dma_len(&chunk->mem[i]), + chunk->buf[i], + sg_dma_address(&chunk->mem[i])); + kfree(chunk); + } + + kfree(hem); +} + +static struct udma_hem *udma_alloc_hem(struct udma_dev *udma_dev, int npages, + uint64_t hem_alloc_size, gfp_t gfp_mask) +{ + struct udma_hem_chunk *chunk = NULL; + struct scatterlist *mem; + struct udma_hem *hem; + int pages = npages; + int order; + void *buf; + + hem = kmalloc(sizeof(*hem), gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); + if (!hem) + return NULL; + + refcount_set(&hem->refcount, 0); + INIT_LIST_HEAD(&hem->chunk_list); + + order = get_order(hem_alloc_size); + + chunk = kmalloc(sizeof(*chunk), gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); + if (!chunk) + goto fail; + + sg_init_table(chunk->mem, UDMA_HEM_CHUNK_LEN); + chunk->npages = 0; + chunk->nsg = 0; + memset(chunk->buf, 0, sizeof(chunk->buf)); + list_add_tail(&chunk->list, &hem->chunk_list); + while (pages > 0) { + while (1 << order > pages) + --order; + mem = &chunk->mem[chunk->npages]; + buf = dma_alloc_coherent(udma_dev->dev, PAGE_SIZE << order, + &sg_dma_address(mem), gfp_mask); + if (!buf) + goto fail; + + chunk->buf[chunk->npages] = buf; + sg_dma_len(mem) = PAGE_SIZE << order; + + ++chunk->npages; + ++chunk->nsg; + pages -= 1 << order; + } + + return hem; + +fail: + udma_free_hem(udma_dev, hem); + return NULL; +} + +static int calc_hem_config(struct udma_dev *udma_dev, + struct udma_hem_table *table, uint64_t obj, + struct udma_hem_mhop *mhop, + struct udma_hem_index *index) +{ + struct device *dev = udma_dev->dev; + uint32_t l0_idx, l1_idx, l2_idx; + uint64_t mhop_obj = obj; + uint32_t chunk_ba_num; + uint32_t bt_num; + int ret; + + ret = udma_calc_hem_mhop(udma_dev, table, &mhop_obj, mhop); + if (ret) + return ret; + + l0_idx = mhop->l0_idx; + l1_idx = mhop->l1_idx; + l2_idx = mhop->l2_idx; + chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; + bt_num = udma_get_bt_num(table->type, mhop->hop_num); + switch (bt_num) { + case 3: + index->l1 = l0_idx * chunk_ba_num + l1_idx; + index->l0 = l0_idx; + index->buf = l0_idx * chunk_ba_num * chunk_ba_num + + l1_idx * chunk_ba_num + l2_idx; + break; + case 2: + index->l0 = l0_idx; + index->buf = l0_idx * chunk_ba_num + l1_idx; + break; + case 1: + index->buf = l0_idx; + break; + default: + dev_err(dev, "table %u not support mhop.hop_num = %u!\n", + table->type, mhop->hop_num); + return -EINVAL; + } + + if (unlikely(index->buf >= table->num_hem)) { + dev_err(dev, "table %u exceed hem limt idx %llu, max %llu!\n", + table->type, index->buf, table->num_hem); + return -EINVAL; + } + + return 0; +} + +static void free_mhop_hem(struct udma_dev *udma_dev, + struct udma_hem_table *table, + struct udma_hem_mhop *mhop, + struct udma_hem_index *index) +{ + uint32_t bt_size = mhop->bt_chunk_size; + struct device *dev = udma_dev->dev; + + if (index->inited & HEM_INDEX_BUF) { + udma_free_hem(udma_dev, table->hem[index->buf]); + table->hem[index->buf] = NULL; + } + + if (index->inited & HEM_INDEX_L1) { + dma_free_coherent(dev, bt_size, table->bt_l1[index->l1], + table->bt_l1_dma_addr[index->l1]); + table->bt_l1[index->l1] = NULL; + } + + if (index->inited & HEM_INDEX_L0) { + dma_free_coherent(dev, bt_size, table->bt_l0[index->l0], + table->bt_l0_dma_addr[index->l0]); + table->bt_l0[index->l0] = NULL; + } +} + +static int alloc_mhop_hem(struct udma_dev *udma_dev, + struct udma_hem_table *table, + struct udma_hem_mhop *mhop, + struct udma_hem_index *index) +{ + uint32_t bt_size = mhop->bt_chunk_size; + struct device *dev = udma_dev->dev; + struct udma_hem_iter iter; + uint64_t bt_ba; + uint32_t size; + gfp_t flag; + int ret; + + /* alloc L1 BA's chunk */ + if ((check_whether_bt_num_3(table->type, mhop->hop_num) || + check_whether_bt_num_2(table->type, mhop->hop_num)) && + !table->bt_l0[index->l0]) { + table->bt_l0[index->l0] = dma_alloc_coherent(dev, bt_size, + &table->bt_l0_dma_addr[index->l0], + GFP_KERNEL); + if (!table->bt_l0[index->l0]) { + ret = -ENOMEM; + goto out; + } + index->inited |= HEM_INDEX_L0; + } + + /* alloc L2 BA's chunk */ + if (check_whether_bt_num_3(table->type, mhop->hop_num) && + !table->bt_l1[index->l1]) { + table->bt_l1[index->l1] = dma_alloc_coherent(dev, bt_size, + &table->bt_l1_dma_addr[index->l1], + GFP_KERNEL); + if (!table->bt_l1[index->l1]) { + ret = -ENOMEM; + goto err_alloc_hem; + } + index->inited |= HEM_INDEX_L1; + *(table->bt_l0[index->l0] + mhop->l1_idx) = + table->bt_l1_dma_addr[index->l1]; + } + + /* + * alloc buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC. + * alloc bt space chunk for MTT/CQE. + */ + size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size; + flag = GFP_KERNEL | __GFP_NOWARN; + table->hem[index->buf] = udma_alloc_hem(udma_dev, size >> PAGE_SHIFT, + size, flag); + if (!table->hem[index->buf]) { + ret = -ENOMEM; + goto err_alloc_hem; + } + + index->inited |= HEM_INDEX_BUF; + udma_hem_first(table->hem[index->buf], &iter); + bt_ba = udma_hem_addr(&iter); + if (table->type < HEM_TYPE_MTT) { + if (mhop->hop_num == 2) + *(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba; + else if (mhop->hop_num == 1) + *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; + } else if (mhop->hop_num == 2) { + *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; + } + + return 0; +err_alloc_hem: + free_mhop_hem(udma_dev, table, mhop, index); +out: + return ret; +} + +static int set_mhop_hem(struct udma_dev *udma_dev, + struct udma_hem_table *table, uint64_t obj, + struct udma_hem_mhop *mhop, + struct udma_hem_index *index) +{ + struct device *dev = udma_dev->dev; + int step_idx; + int ret = 0; + + if (index->inited & HEM_INDEX_L0) { + ret = udma_dev->hw->set_hem(udma_dev, table, obj, 0); + if (ret) { + dev_err(dev, "set HEM step 0 failed!\n"); + goto out; + } + } + + if (index->inited & HEM_INDEX_L1) { + ret = udma_dev->hw->set_hem(udma_dev, table, obj, 1); + if (ret) { + dev_err(dev, "set HEM step 1 failed!\n"); + goto out; + } + } + + if (index->inited & HEM_INDEX_BUF) { + if (mhop->hop_num == UDMA_HOP_NUM_0) + step_idx = 0; + else + step_idx = mhop->hop_num; + ret = udma_dev->hw->set_hem(udma_dev, table, obj, step_idx); + if (ret) + dev_err(dev, "set HEM step last failed!\n"); + } +out: + return ret; +} + +static int udma_table_mhop_get(struct udma_dev *udma_dev, + struct udma_hem_table *table, + uint64_t obj) +{ + struct device *dev = udma_dev->dev; + struct udma_hem_index index = {}; + struct udma_hem_mhop mhop = {}; + int ret; + + ret = calc_hem_config(udma_dev, table, obj, &mhop, &index); + if (ret) { + dev_err(dev, "calc hem config failed!\n"); + return ret; + } + + mutex_lock(&table->mutex); + if (table->hem[index.buf]) { + refcount_inc(&table->hem[index.buf]->refcount); + goto out; + } + + ret = alloc_mhop_hem(udma_dev, table, &mhop, &index); + if (ret) { + dev_err(dev, "alloc mhop hem failed!\n"); + goto out; + } + + /* set HEM base address to hardware */ + if (table->type < HEM_TYPE_MTT) { + ret = set_mhop_hem(udma_dev, table, obj, &mhop, &index); + if (ret) { + dev_err(dev, "set HEM address to HW failed!\n"); + goto err_alloc; + } + } + + refcount_set(&table->hem[index.buf]->refcount, 1); + goto out; + +err_alloc: + free_mhop_hem(udma_dev, table, &mhop, &index); +out: + mutex_unlock(&table->mutex); + return ret; +} + +int udma_table_get(struct udma_dev *udma_dev, + struct udma_hem_table *table, uint64_t obj) +{ + struct device *dev = udma_dev->dev; + int ret = 0; + uint64_t i; + + if (udma_check_whether_mhop(udma_dev, table->type)) + return udma_table_mhop_get(udma_dev, table, obj); + + i = obj / (table->table_chunk_size / table->obj_size); + + mutex_lock(&table->mutex); + + if (table->hem[i]) { + refcount_inc(&table->hem[i]->refcount); + goto out; + } + + table->hem[i] = udma_alloc_hem(udma_dev, + table->table_chunk_size >> PAGE_SHIFT, + table->table_chunk_size, + GFP_KERNEL | __GFP_NOWARN); + if (!table->hem[i]) { + ret = -ENOMEM; + goto out; + } + + /* Set HEM base address(128K/page, pa) to Hardware */ + if (udma_dev->hw->set_hem(udma_dev, table, obj, HEM_HOP_STEP_DIRECT)) { + udma_free_hem(udma_dev, table->hem[i]); + table->hem[i] = NULL; + ret = -ENODEV; + dev_err(dev, "set HEM base address to HW failed.\n"); + goto out; + } + + refcount_set(&table->hem[i]->refcount, 1); +out: + mutex_unlock(&table->mutex); + return ret; +} + +static void clear_mhop_hem(struct udma_dev *udma_dev, + struct udma_hem_table *table, uint64_t obj, + struct udma_hem_mhop *mhop, + struct udma_hem_index *index) +{ + struct device *dev = udma_dev->dev; + uint32_t hop_num = mhop->hop_num; + uint32_t chunk_ba_num; + int step_idx; + + index->inited = HEM_INDEX_BUF; + chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; + if (check_whether_bt_num_2(table->type, hop_num)) { + if (udma_check_hem_null(table->hem, index->buf, + chunk_ba_num, table->num_hem)) + index->inited |= HEM_INDEX_L0; + } else if (check_whether_bt_num_3(table->type, hop_num)) { + if (udma_check_hem_null(table->hem, index->buf, + chunk_ba_num, table->num_hem)) { + index->inited |= HEM_INDEX_L1; + if (udma_check_bt_null(table->bt_l1, index->l1, + chunk_ba_num)) + index->inited |= HEM_INDEX_L0; + } + } + + if (table->type < HEM_TYPE_MTT) { + if (hop_num == UDMA_HOP_NUM_0) + step_idx = 0; + else + step_idx = hop_num; + + if (udma_dev->hw->clear_hem(udma_dev, table, obj, step_idx)) + dev_err(dev, "failed to clear hop%u HEM.\n", hop_num); + + if (index->inited & HEM_INDEX_L1) + if (udma_dev->hw->clear_hem(udma_dev, table, obj, 1)) + dev_err(dev, "failed to clear HEM step 1.\n"); + + if (index->inited & HEM_INDEX_L0) + if (udma_dev->hw->clear_hem(udma_dev, table, obj, 0)) + dev_err(dev, "failed to clear HEM step 0.\n"); + } +} + +static void udma_table_mhop_put(struct udma_dev *udma_dev, + struct udma_hem_table *table, uint64_t obj, + int check_refcount) +{ + struct device *dev = udma_dev->dev; + struct udma_hem_index index = {}; + struct udma_hem_mhop mhop = {}; + int ret; + + ret = calc_hem_config(udma_dev, table, obj, &mhop, &index); + if (ret) { + dev_err(dev, "calc hem config failed!\n"); + return; + } + + if (!check_refcount) + mutex_lock(&table->mutex); + else if (!refcount_dec_and_mutex_lock(&table->hem[index.buf]->refcount, + &table->mutex)) + return; + + clear_mhop_hem(udma_dev, table, obj, &mhop, &index); + free_mhop_hem(udma_dev, table, &mhop, &index); + + mutex_unlock(&table->mutex); +} + +void udma_table_put(struct udma_dev *udma_dev, + struct udma_hem_table *table, uint64_t obj) +{ + struct device *dev = udma_dev->dev; + uint64_t i; + + if (udma_check_whether_mhop(udma_dev, table->type)) { + udma_table_mhop_put(udma_dev, table, obj, 1); + return; + } + + i = obj / (table->table_chunk_size / table->obj_size); + + if (!refcount_dec_and_mutex_lock(&table->hem[i]->refcount, + &table->mutex)) + return; + + if (udma_dev->hw->clear_hem(udma_dev, table, obj, HEM_HOP_STEP_DIRECT)) + dev_warn(dev, "failed to clear HEM base address.\n"); + + udma_free_hem(udma_dev, table->hem[i]); + table->hem[i] = NULL; + + mutex_unlock(&table->mutex); +} + +void *udma_table_find(struct udma_dev *udma_dev, + struct udma_hem_table *table, + uint64_t obj, dma_addr_t *dma_handle) +{ + struct udma_hem_chunk *chunk; + struct udma_hem_mhop mhop; + uint64_t mhop_obj = obj; + uint64_t obj_per_chunk; + int offset, dma_offset; + struct udma_hem *hem; + uint32_t hem_idx = 0; + uint64_t idx_offset; + void *addr = NULL; + uint32_t length; + uint32_t i, j; + + mutex_lock(&table->mutex); + + if (!udma_check_whether_mhop(udma_dev, table->type)) { + obj_per_chunk = table->table_chunk_size / table->obj_size; + hem = table->hem[obj / obj_per_chunk]; + idx_offset = obj % obj_per_chunk; + dma_offset = offset = idx_offset * table->obj_size; + } else { + /* 8 bytes per BA and 8 BA per segment */ + uint32_t seg_size = 64; + + if (udma_calc_hem_mhop(udma_dev, table, &mhop_obj, &mhop)) + goto out; + /* mtt mhop */ + i = mhop.l0_idx; + j = mhop.l1_idx; + if (mhop.hop_num == 2) + hem_idx = i * (mhop.bt_chunk_size / BA_BYTE_LEN) + j; + else if (mhop.hop_num == 1 || + mhop.hop_num == UDMA_HOP_NUM_0) + hem_idx = i; + + hem = table->hem[hem_idx]; + dma_offset = offset = obj * seg_size % mhop.bt_chunk_size; + if (mhop.hop_num == 2) + dma_offset = offset = 0; + } + + if (!hem) + goto out; + + list_for_each_entry(chunk, &hem->chunk_list, list) { + for (i = 0; (int)i < chunk->npages; ++i) { + length = sg_dma_len(&chunk->mem[i]); + if (dma_handle && dma_offset >= 0) { + *dma_handle = length > (uint32_t)dma_offset ? sg_dma_address( + &chunk->mem[i]) + dma_offset : *dma_handle; + dma_offset -= length; + } + + if (length > (uint32_t)offset) { + addr = (char *)chunk->buf[i] + offset; + goto out; + } + offset -= length; + } + } + +out: + mutex_unlock(&table->mutex); + return addr; +} + +int udma_init_hem_table(struct udma_dev *udma_dev, + struct udma_hem_table *table, uint32_t type, + uint64_t obj_size, uint64_t nobj) +{ + uint64_t obj_per_chunk; + uint64_t num_hem; + + if (!udma_check_whether_mhop(udma_dev, type)) { + table->table_chunk_size = udma_dev->caps.chunk_sz; + obj_per_chunk = table->table_chunk_size / obj_size; + num_hem = DIV_ROUND_UP(nobj, obj_per_chunk); + + table->hem = kcalloc(num_hem, sizeof(*table->hem), + GFP_KERNEL); + if (!table->hem) + return -ENOMEM; + } else { + struct udma_hem_mhop mhop = {}; + uint64_t buf_chunk_size; + uint64_t bt_chunk_size; + uint64_t bt_chunk_num; + uint64_t num_bt_l0; + uint32_t hop_num; + int ret; + + ret = get_hem_table_config(udma_dev, &mhop, type); + if (ret) + return ret; + + buf_chunk_size = mhop.buf_chunk_size; + bt_chunk_size = mhop.bt_chunk_size; + num_bt_l0 = mhop.ba_l0_num; + hop_num = mhop.hop_num; + + obj_per_chunk = buf_chunk_size / obj_size; + num_hem = DIV_ROUND_UP(nobj, obj_per_chunk); + bt_chunk_num = bt_chunk_size / BA_BYTE_LEN; + + if (type >= HEM_TYPE_MTT) + num_bt_l0 = bt_chunk_num; + + table->hem = kcalloc(num_hem, sizeof(*table->hem), + GFP_KERNEL); + if (!table->hem) + goto err_kcalloc_hem_buf; + + if (check_whether_bt_num_3(type, hop_num)) { + uint64_t num_bt_l1; + + num_bt_l1 = DIV_ROUND_UP(num_hem, bt_chunk_num); + table->bt_l1 = kcalloc(num_bt_l1, sizeof(*table->bt_l1), + GFP_KERNEL); + if (!table->bt_l1) + goto err_kcalloc_bt_l1; + + table->bt_l1_dma_addr = kcalloc(num_bt_l1, + sizeof(*table->bt_l1_dma_addr), + GFP_KERNEL); + + if (!table->bt_l1_dma_addr) + goto err_kcalloc_l1_dma; + } + + if (check_whether_bt_num_2(type, hop_num) || + check_whether_bt_num_3(type, hop_num)) { + table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0), + GFP_KERNEL); + if (!table->bt_l0) + goto err_kcalloc_bt_l0; + + table->bt_l0_dma_addr = kcalloc(num_bt_l0, + sizeof(*table->bt_l0_dma_addr), + GFP_KERNEL); + if (!table->bt_l0_dma_addr) + goto err_kcalloc_l0_dma; + } + } + + table->type = type; + table->num_hem = num_hem; + table->obj_size = obj_size; + mutex_init(&table->mutex); + + return 0; + +err_kcalloc_l0_dma: + kfree(table->bt_l0); + table->bt_l0 = NULL; + +err_kcalloc_bt_l0: + kfree(table->bt_l1_dma_addr); + table->bt_l1_dma_addr = NULL; + +err_kcalloc_l1_dma: + kfree(table->bt_l1); + table->bt_l1 = NULL; + +err_kcalloc_bt_l1: + kfree(table->hem); + table->hem = NULL; + +err_kcalloc_hem_buf: + return -ENOMEM; +} + +static void udma_cleanup_mhop_hem_table(struct udma_dev *udma_dev, + struct udma_hem_table *table) +{ + struct udma_hem_mhop mhop; + uint32_t buf_chunk_size; + uint64_t obj; + uint32_t i; + + if (udma_calc_hem_mhop(udma_dev, table, NULL, &mhop)) + return; + buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size : + mhop.bt_chunk_size; + + for (i = 0; i < table->num_hem; ++i) { + obj = i * buf_chunk_size / table->obj_size; + if (table->hem[i]) + udma_table_mhop_put(udma_dev, table, obj, 0); + } + + kfree(table->hem); + table->hem = NULL; + kfree(table->bt_l1); + table->bt_l1 = NULL; + kfree(table->bt_l1_dma_addr); + table->bt_l1_dma_addr = NULL; + kfree(table->bt_l0); + table->bt_l0 = NULL; + kfree(table->bt_l0_dma_addr); + table->bt_l0_dma_addr = NULL; +} + +void udma_cleanup_hem_table(struct udma_dev *udma_dev, + struct udma_hem_table *table) +{ + struct device *dev = udma_dev->dev; + uint64_t i; + + if (udma_check_whether_mhop(udma_dev, table->type)) { + udma_cleanup_mhop_hem_table(udma_dev, table); + return; + } + + for (i = 0; i < table->num_hem; ++i) + if (table->hem[i]) { + if (udma_dev->hw->clear_hem(udma_dev, table, + i * table->table_chunk_size + / table->obj_size, 0)) + dev_err(dev, "Clear HEM base address failed.\n"); + + udma_free_hem(udma_dev, table->hem[i]); + } + + kfree(table->hem); +} + +static struct udma_hem_item * +hem_list_alloc_item(struct udma_dev *udma_dev, int start, int end, int count, + bool exist_bt) +{ + struct udma_hem_item *hem; + + hem = kzalloc(sizeof(*hem), GFP_KERNEL); + if (!hem) + return NULL; + + if (exist_bt) { + hem->addr = dma_alloc_coherent(udma_dev->dev, + count * BA_BYTE_LEN, + &hem->dma_addr, GFP_KERNEL); + if (!hem->addr) { + kfree(hem); + return NULL; + } + } + + hem->count = count; + hem->start = start; + hem->end = end; + INIT_LIST_HEAD(&hem->list); + INIT_LIST_HEAD(&hem->sibling); + + return hem; +} + +static void hem_list_free_item(struct udma_dev *udma_dev, + struct udma_hem_item *hem, bool exist_bt) +{ + if (exist_bt) + dma_free_coherent(udma_dev->dev, hem->count * BA_BYTE_LEN, + hem->addr, hem->dma_addr); + kfree(hem); +} + +static void hem_list_free_all(struct udma_dev *udma_dev, + struct list_head *head, bool exist_bt) +{ + struct udma_hem_item *hem, *temp_hem; + + list_for_each_entry_safe(hem, temp_hem, head, list) { + list_del(&hem->list); + hem_list_free_item(udma_dev, hem, exist_bt); + } +} + +static void hem_list_link_bt(struct udma_dev *udma_dev, void *base_addr, + uint64_t table_addr) +{ + *(uint64_t *)(base_addr) = table_addr; +} + +/* assign L0 table address to hem from root bt */ +static void hem_list_assign_bt(struct udma_dev *udma_dev, + struct udma_hem_item *hem, void *cpu_addr, + uint64_t phy_addr) +{ + hem->addr = cpu_addr; + hem->dma_addr = (dma_addr_t)phy_addr; +} + +static inline bool hem_list_page_is_in_range(struct udma_hem_item *hem, + int offset) +{ + return (hem->start <= offset && offset <= hem->end); +} + +static struct udma_hem_item *hem_list_search_item(struct list_head *ba_list, + int page_offset) +{ + struct udma_hem_item *found = NULL; + struct udma_hem_item *hem; + + list_for_each_entry(hem, ba_list, list) { + if (hem_list_page_is_in_range(hem, page_offset)) { + found = hem; + break; + } + } + + return found; +} + +static bool hem_list_is_bottom_bt(int hopnum, int bt_level) +{ + /* + * hopnum base address table levels + * 0 L0(buf) + * 1 L0 -> buf + * 2 L0 -> L1 -> buf + * 3 L0 -> L1 -> L2 -> buf + */ + return bt_level >= (hopnum ? hopnum - 1 : hopnum); +} + +/** + * calc base address entries num + * @hopnum: num of mutihop addressing + * @bt_level: base address table level + * @unit: ba entries per bt page + */ +static uint32_t hem_list_calc_ba_range(int hopnum, int bt_level, int unit) +{ + uint32_t step; + int max; + int i; + + if (hopnum <= bt_level) + return 0; + /* + * hopnum bt_level range + * 1 0 unit + * ------------ + * 2 0 unit * unit + * 2 1 unit + * ------------ + * 3 0 unit * unit * unit + * 3 1 unit * unit + * 3 2 unit + */ + step = 1; + max = hopnum - bt_level; + for (i = 0; i < max; i++) + step = step * unit; + + return step; +} + +/** + * calc the root ba entries which could cover all regions + * @regions: buf region array + * @region_cnt: array size of @regions + * @unit: ba entries per bt page + */ +static int udma_hem_list_calc_root_ba(const struct udma_buf_region *regions, + int region_cnt, int unit) +{ + struct udma_buf_region *r; + int total = 0; + int step; + int i; + + for (i = 0; i < region_cnt; i++) { + r = (struct udma_buf_region *)®ions[i]; + if (r->hopnum > 1) { + step = hem_list_calc_ba_range(r->hopnum, 1, unit); + if (step > 0) + total += (r->count + step - 1) / step; + } else { + total += r->count; + } + } + + return total; +} + +static int hem_list_alloc_mid_bt(struct udma_dev *udma_dev, + const struct udma_buf_region *r, int unit, + uint32_t offset, struct list_head *mid_bt, + struct list_head *btm_bt) +{ + struct udma_hem_item *hem_ptrs[UDMA_MAX_BT_LEVEL] = { NULL }; + struct list_head temp_list[UDMA_MAX_BT_LEVEL]; + struct udma_hem_item *cur, *pre; + const int hopnum = r->hopnum; + int start_aligned; + uint32_t step; + int distance; + int ret = 0; + int max_ofs; + int level; + int end; + + if (hopnum <= 1) + return 0; + + if (hopnum > UDMA_MAX_BT_LEVEL) { + dev_err(udma_dev->dev, "invalid hopnum %d!\n", hopnum); + return -EINVAL; + } + + if (offset < r->offset) { + dev_err(udma_dev->dev, "invalid offset %d, min %u!\n", + offset, r->offset); + return -EINVAL; + } + + distance = offset - r->offset; + max_ofs = r->offset + r->count - 1; + for (level = 0; level < hopnum; level++) + INIT_LIST_HEAD(&temp_list[level]); + + /* config L1 bt to last bt and link them to corresponding parent */ + for (level = 1; level < hopnum; level++) { + cur = hem_list_search_item(&mid_bt[level], offset); + if (cur) { + hem_ptrs[level] = cur; + continue; + } + + step = hem_list_calc_ba_range(hopnum, level, unit); + if (step < 1) { + ret = -EINVAL; + goto err_exit; + } + + start_aligned = (distance / step) * step + r->offset; + end = min_t(int, start_aligned + step - 1, max_ofs); + cur = hem_list_alloc_item(udma_dev, start_aligned, end, unit, + true); + if (!cur) { + ret = -ENOMEM; + goto err_exit; + } + hem_ptrs[level] = cur; + list_add(&cur->list, &temp_list[level]); + if (hem_list_is_bottom_bt(hopnum, level)) + list_add(&cur->sibling, &temp_list[0]); + + /* link bt to parent bt */ + if (level > 1) { + pre = hem_ptrs[level - 1]; + step = (cur->start - pre->start) / step * BA_BYTE_LEN; + hem_list_link_bt(udma_dev, (char *)pre->addr + step, + cur->dma_addr); + } + } + + list_splice(&temp_list[0], btm_bt); + for (level = 1; level < hopnum; level++) + list_splice(&temp_list[level], &mid_bt[level]); + + return 0; + +err_exit: + for (level = 1; level < hopnum; level++) + hem_list_free_all(udma_dev, &temp_list[level], true); + + return ret; +} + +static struct udma_hem_item * +alloc_root_hem(struct udma_dev *udma_dev, int unit, int *max_ba_num, + const struct udma_buf_region *regions, int region_cnt) +{ + const struct udma_buf_region *r; + struct udma_hem_item *hem; + int ba_num; + int offset; + + ba_num = udma_hem_list_calc_root_ba(regions, region_cnt, unit); + if (ba_num < 1) + return ERR_PTR(-ENOMEM); + + if (ba_num > unit) + return ERR_PTR(-ENOBUFS); + + offset = regions[0].offset; + /* indicate to last region */ + r = ®ions[region_cnt - 1]; + hem = hem_list_alloc_item(udma_dev, offset, r->offset + r->count - 1, + ba_num, true); + if (!hem) + return ERR_PTR(-ENOMEM); + + *max_ba_num = ba_num; + + return hem; +} + +static int alloc_fake_root_bt(struct udma_dev *udma_dev, void *cpu_base, + uint64_t phy_base, + const struct udma_buf_region *r, + struct list_head *branch_head, + struct list_head *leaf_head) +{ + struct udma_hem_item *hem; + + hem = hem_list_alloc_item(udma_dev, r->offset, r->offset + r->count - 1, + r->count, false); + if (!hem) + return -ENOMEM; + + hem_list_assign_bt(udma_dev, hem, cpu_base, phy_base); + list_add(&hem->list, branch_head); + list_add(&hem->sibling, leaf_head); + + return r->count; +} + +static int setup_middle_bt(struct udma_dev *udma_dev, void *cpu_base, + int unit, const struct udma_buf_region *r, + const struct list_head *branch_head) +{ + struct udma_hem_item *hem; + int total = 0; + int offset; + int step; + + step = hem_list_calc_ba_range(r->hopnum, 1, unit); + if (step < 1) + return -EINVAL; + + /* if exist mid bt, link L1 to L0 */ + list_for_each_entry(hem, branch_head, list) { + offset = (hem->start - r->offset) / step * BA_BYTE_LEN; + hem_list_link_bt(udma_dev, (char *)cpu_base + offset, + hem->dma_addr); + total++; + } + + return total; +} + +static int +setup_root_hem(struct udma_dev *udma_dev, struct udma_hem_list *hem_list, + int unit, int max_ba_num, struct udma_hem_head *head, + const struct udma_buf_region *regions, int region_cnt) +{ + const struct udma_buf_region *r; + struct udma_hem_item *root_hem; + uint64_t phy_base; + void *cpu_base; + int i, total; + int ret; + + root_hem = list_first_entry(&head->root, + struct udma_hem_item, list); + if (!root_hem) + return -ENOMEM; + + total = 0; + for (i = 0; i < region_cnt && total < max_ba_num; i++) { + r = ®ions[i]; + if (!r->count) + continue; + + /* all regions's mid[x][0] shared the root_bt's trunk */ + cpu_base = (char *)root_hem->addr + total * BA_BYTE_LEN; + phy_base = root_hem->dma_addr + total * BA_BYTE_LEN; + + /* if hopnum is 0 or 1, cut a new fake hem from the root bt + * which's address share to all regions. + */ + if (hem_list_is_bottom_bt(r->hopnum, 0)) + ret = alloc_fake_root_bt(udma_dev, cpu_base, phy_base, + r, &head->branch[i], + &head->leaf); + else + ret = setup_middle_bt(udma_dev, cpu_base, unit, r, + &hem_list->mid_bt[i][1]); + + if (ret < 0) + return ret; + + total += ret; + } + + list_splice(&head->leaf, &hem_list->btm_bt); + list_splice(&head->root, &hem_list->root_bt); + for (i = 0; i < region_cnt; i++) + list_splice(&head->branch[i], &hem_list->mid_bt[i][0]); + + return 0; +} + +static int hem_list_alloc_root_bt(struct udma_dev *udma_dev, + struct udma_hem_list *hem_list, int unit, + const struct udma_buf_region *regions, + int region_cnt) +{ + struct udma_hem_item *root_hem; + struct udma_hem_head head; + int max_ba_num; + int ret; + int i; + + /* Existed in hem list */ + root_hem = hem_list_search_item(&hem_list->root_bt, regions[0].offset); + if (root_hem) + return 0; + + max_ba_num = 0; + root_hem = alloc_root_hem(udma_dev, unit, &max_ba_num, regions, + region_cnt); + if (IS_ERR(root_hem)) + return PTR_ERR(root_hem); + + /* List head for storing all allocated HEM items */ + INIT_LIST_HEAD(&head.root); + INIT_LIST_HEAD(&head.leaf); + for (i = 0; i < region_cnt; i++) + INIT_LIST_HEAD(&head.branch[i]); + + hem_list->root_ba = root_hem->dma_addr; + list_add(&root_hem->list, &head.root); + ret = setup_root_hem(udma_dev, hem_list, unit, max_ba_num, &head, + regions, region_cnt); + if (ret) { + for (i = 0; i < region_cnt; i++) + hem_list_free_all(udma_dev, &head.branch[i], false); + + hem_list_free_all(udma_dev, &head.root, true); + } + + return ret; +} + +static void udma_hem_list_release(struct udma_dev *udma_dev, + struct udma_hem_list *hem_list) +{ + int i, j; + + for (i = 0; i < UDMA_MAX_BT_REGION; i++) + for (j = 0; j < UDMA_MAX_BT_LEVEL; j++) + hem_list_free_all(udma_dev, &hem_list->mid_bt[i][j], + j != 0); + + hem_list_free_all(udma_dev, &hem_list->root_bt, true); + INIT_LIST_HEAD(&hem_list->btm_bt); + hem_list->root_ba = 0; +} + +/* construct the base address table and link them by address hop config */ +static int udma_hem_list_request(struct udma_dev *udma_dev, + struct udma_hem_list *hem_list, + const struct udma_buf_region *regions, + int region_cnt, uint32_t bt_pg_shift) +{ + const struct udma_buf_region *r; + uint32_t ofs; + uint32_t end; + int unit; + int ret; + int i; + + if (region_cnt > UDMA_MAX_BT_REGION) { + dev_err(udma_dev->dev, "invalid region region_cnt %d!\n", + region_cnt); + return -EINVAL; + } + + unit = (1 << bt_pg_shift) / BA_BYTE_LEN; + for (i = 0; i < region_cnt; i++) { + r = ®ions[i]; + if (!r->count) + continue; + + end = r->offset + r->count; + for (ofs = r->offset; ofs < end; ofs += unit) { + ret = hem_list_alloc_mid_bt(udma_dev, r, unit, ofs, + hem_list->mid_bt[i], + &hem_list->btm_bt); + if (ret) { + dev_err(udma_dev->dev, + "alloc hem trunk fail ret = %d!\n", + ret); + goto err_alloc; + } + } + } + + ret = hem_list_alloc_root_bt(udma_dev, hem_list, unit, regions, + region_cnt); + if (ret) + dev_err(udma_dev->dev, "alloc hem root fail ret = %d!\n", ret); + else + return 0; + +err_alloc: + udma_hem_list_release(udma_dev, hem_list); + + return ret; +} + +static void udma_hem_list_init(struct udma_hem_list *hem_list) +{ + int i, j; + + INIT_LIST_HEAD(&hem_list->root_bt); + INIT_LIST_HEAD(&hem_list->btm_bt); + for (i = 0; i < UDMA_MAX_BT_REGION; i++) + for (j = 0; j < UDMA_MAX_BT_LEVEL; j++) + INIT_LIST_HEAD(&hem_list->mid_bt[i][j]); +} + +static void *udma_hem_list_find_mtt(struct udma_dev *udma_dev, + struct udma_hem_list *hem_list, int offset, + int *mtt_cnt, uint64_t *phy_addr) +{ + struct list_head *head = &hem_list->btm_bt; + struct udma_hem_item *hem; + int relative_offset = 0; + void *cpu_base = NULL; + uint64_t phy_base = 0; + + if (IS_ERR_OR_NULL(head->next)) + return cpu_base; + list_for_each_entry(hem, head, sibling) { + if (hem_list_page_is_in_range(hem, offset)) { + relative_offset = offset - hem->start; + cpu_base = (char *)hem->addr + relative_offset * + BA_BYTE_LEN; + phy_base = hem->dma_addr + relative_offset * + BA_BYTE_LEN; + relative_offset = hem->end + 1 - offset; + break; + } + } + + if (mtt_cnt) + *mtt_cnt = relative_offset; + + if (phy_addr) + *phy_addr = phy_base; + + return cpu_base; +} + +static inline bool mtr_has_mtt(struct udma_buf_attr *attr) +{ + uint32_t i; + + for (i = 0; i < attr->region_count; i++) + if (attr->region[i].hopnum != UDMA_HOP_NUM_0 && + attr->region[i].hopnum > 0) + return true; + + /* Mtr only has one root base address, when hopnum 0 means root base + * address equals the first buffer address. So all alloced memory must + * in a continuous space accessed by direct mode. + */ + return false; +} + +static inline size_t mtr_bufs_size(struct udma_buf_attr *attr) +{ + size_t size = 0; + uint32_t i; + + for (i = 0; i < attr->region_count; i++) + size += attr->region[i].size; + + return size; +} + +static bool need_split_huge_page(struct udma_hem_cfg *cfg) +{ + /* When HEM buffer uses 0-level addressing, the page size is + * equal to the whole buffer size. If the current MTR has multiple + * regions, we split the buffer into small pages(4k, required by + * UDMA). These pages will be used in multiple regions. + */ + return cfg->is_direct && cfg->region_count > 1; +} + +static int mtr_init_buf_cfg(struct udma_dev *udma_dev, + struct udma_buf_attr *attr, + struct udma_hem_cfg *cfg, + uint32_t *buf_page_shift, int unalinged_size) +{ + uint32_t page_count, region_count; + struct udma_buf_region *r; + int first_region_pad; + uint32_t page_shift; + size_t buf_size; + + /* If mtt is disabled, all pages must be within a continuous range */ + cfg->is_direct = !mtr_has_mtt(attr); + buf_size = mtr_bufs_size(attr); + if (need_split_huge_page(cfg)) { + /* When HEM buffer uses 0-level addressing, the page size is + * equal to the whole buffer size, and we split the buffer into + * small pages which is used to check whether the adjacent + * units are in the continuous space and its size is fixed to + * 4K based on hns ROCEE's requirement. + */ + page_shift = UDMA_HW_PAGE_SHIFT; + + cfg->buf_pg_count = 1; + cfg->buf_pg_shift = UDMA_HW_PAGE_SHIFT + + order_base_2(DIV_ROUND_UP(buf_size, UDMA_PAGE_SIZE)); + first_region_pad = 0; + } else { + page_shift = attr->page_shift; + cfg->buf_pg_count = DIV_ROUND_UP(buf_size + unalinged_size, + 1 << page_shift); + cfg->buf_pg_shift = page_shift; + first_region_pad = unalinged_size; + } + + /* Convert buffer size to page index and page count for each region. + * The buffer's offset needs to be appended to the first region. + */ + for (page_count = 0, region_count = 0; region_count < attr->region_count && + region_count < ARRAY_SIZE(cfg->region); region_count++) { + r = &cfg->region[region_count]; + r->offset = page_count; + buf_size = udma_hw_page_align(attr->region[region_count].size + + first_region_pad); + r->count = DIV_ROUND_UP(buf_size, 1 << page_shift); + first_region_pad = 0; + page_count += r->count; + r->hopnum = to_udma_hem_hopnum(attr->region[region_count].hopnum, + r->count); + } + + cfg->region_count = region_count; + *buf_page_shift = page_shift; + + return page_count; +} + +static int mtr_alloc_mtt(struct udma_dev *udma_dev, struct udma_mtr *mtr, + uint32_t ba_pg_shift) +{ + struct udma_hem_cfg *cfg = &mtr->hem_cfg; + int ret; + + udma_hem_list_init(&mtr->hem_list); + if (!cfg->is_direct) { + ret = udma_hem_list_request(udma_dev, &mtr->hem_list, + cfg->region, cfg->region_count, + ba_pg_shift); + if (ret) + return ret; + cfg->root_ba = mtr->hem_list.root_ba; + cfg->ba_pg_shift = ba_pg_shift; + } else { + cfg->ba_pg_shift = cfg->buf_pg_shift; + } + + return 0; +} + +struct udma_buf *udma_buf_alloc(struct udma_dev *udma_dev, uint32_t size, + uint32_t page_shift, uint32_t flags) +{ + uint32_t trunk_size, page_size, alloced_size; + struct udma_buf_list *trunks; + struct udma_buf *buf; + uint32_t ntrunk, i; + gfp_t gfp_flags; + + /* The minimum shift of the page accessed by hw is UDMA_PAGE_SHIFT */ + if (WARN_ON(page_shift < UDMA_HW_PAGE_SHIFT)) + return ERR_PTR(-EINVAL); + + gfp_flags = (flags & UDMA_BUF_NOSLEEP) ? GFP_ATOMIC : GFP_KERNEL; + buf = kzalloc(sizeof(*buf), gfp_flags); + if (!buf) + return ERR_PTR(-ENOMEM); + + buf->page_shift = page_shift; + page_size = 1 << buf->page_shift; + + /* Calc the trunk size and num by required size and page_shift */ + if (flags & UDMA_BUF_DIRECT) { + buf->trunk_shift = order_base_2(ALIGN(size, PAGE_SIZE)); + ntrunk = 1; + } else { + buf->trunk_shift = order_base_2(ALIGN(page_size, PAGE_SIZE)); + ntrunk = DIV_ROUND_UP(size, 1 << buf->trunk_shift); + } + + trunks = kcalloc(ntrunk, sizeof(*trunks), gfp_flags); + if (!trunks) { + kfree(buf); + return ERR_PTR(-ENOMEM); + } + + trunk_size = 1 << buf->trunk_shift; + alloced_size = 0; + for (i = 0; i < ntrunk; i++) { + trunks[i].buf = dma_alloc_coherent(udma_dev->dev, trunk_size, + &trunks[i].map, gfp_flags); + if (!trunks[i].buf) + break; + + alloced_size += trunk_size; + } + + buf->ntrunks = i; + + /* In nofail mode, it's only failed when the alloced size is 0 */ + if ((flags & UDMA_BUF_NOFAIL) ? i == 0 : i != ntrunk) { + for (i = 0; i < buf->ntrunks; i++) + dma_free_coherent(udma_dev->dev, trunk_size, + trunks[i].buf, trunks[i].map); + + kfree(trunks); + kfree(buf); + return ERR_PTR(-ENOMEM); + } + + buf->npages = DIV_ROUND_UP(alloced_size, page_size); + buf->trunk_list = trunks; + + return buf; +} + +static int mtr_alloc_bufs(struct udma_dev *udma_dev, struct udma_mtr *mtr, + struct udma_buf_attr *buf_attr, + uint64_t user_addr, bool is_user) +{ + struct ubcore_device *ubcore_dev = &udma_dev->ub_dev; + union ubcore_umem_flag access; + size_t total_size; + + total_size = mtr_bufs_size(buf_attr); + + if (is_user) { + mtr->kmem = NULL; + access.bs.non_pin = 0; + access.bs.writable = 1; + mtr->umem = ubcore_umem_get(ubcore_dev, user_addr, total_size, + access); + if (IS_ERR_OR_NULL(mtr->umem)) { + dev_err(udma_dev->dev, + "failed to get umem, ret = %ld.\n", + PTR_ERR(mtr->umem)); + return -ENOMEM; + } + } else { + mtr->umem = NULL; + mtr->kmem = udma_buf_alloc(udma_dev, total_size, + buf_attr->page_shift, + mtr->hem_cfg.is_direct ? + UDMA_BUF_DIRECT : 0); + if (IS_ERR(mtr->kmem)) { + dev_err(udma_dev->dev, + "failed to alloc kmem, ret = %ld.\n", + PTR_ERR(mtr->kmem)); + return PTR_ERR(mtr->kmem); + } + } + + return 0; +} + +int udma_get_umem_bufs(struct udma_dev *udma_dev, dma_addr_t *bufs, + int buf_cnt, struct ubcore_umem *umem, + uint32_t page_shift) +{ + struct scatterlist *sg; + int npage_per_sg; + dma_addr_t addr; + int npage = 0; + int total = 0; + uint32_t k; + int i; + + for_each_sg(umem->sg_head.sgl, sg, umem->sg_head.nents, k) { + npage_per_sg = sg_dma_len(sg) >> page_shift; + for (i = 0; i < npage_per_sg; i++) { + addr = sg_dma_address(sg) + (i << page_shift); + if (addr & ((1 << page_shift) - 1)) { + dev_err(udma_dev->dev, + "Umem addr not align to page_shift %d!\n", + page_shift); + return -EFAULT; + } + + bufs[total++] = addr; + if (total >= buf_cnt) + goto done; + + npage++; + } + } +done: + return total; +} + +static inline int mtr_check_direct_pages(dma_addr_t *pages, int page_cnt, + uint32_t page_shift) +{ + size_t page_sz = 1 << page_shift; + int i; + + for (i = 1; i < page_cnt; i++) + if (pages[i] - pages[i - 1] != page_sz) + return i; + + return 0; +} + +static int mtr_map_region(struct udma_dev *udma_dev, struct udma_mtr *mtr, + struct udma_buf_region *region, dma_addr_t *pages, + int max_count) +{ + int offset, end; + uint64_t *mtts; + uint64_t addr; + int npage = 0; + int count; + int i; + + offset = region->offset; + end = offset + region->count; + while (offset < end && npage < max_count) { + count = 0; + mtts = (uint64_t *)udma_hem_list_find_mtt(udma_dev, + &mtr->hem_list, + offset, &count, NULL); + if (!mtts) + return -ENOBUFS; + + for (i = 0; i < count && npage < max_count; i++) { + addr = pages[npage]; + mtts[i] = cpu_to_le64(addr); + npage++; + } + offset += count; + } + + return npage; +} + +int udma_mtr_map(struct udma_dev *udma_dev, struct udma_mtr *mtr, + dma_addr_t *pages, uint32_t page_count) +{ + struct device *dev = udma_dev->dev; + struct udma_buf_region *r; + uint32_t i, mapped_count; + int ret = 0; + + /* + * Only first page address was used as root ba when hopnum is 0, + * because the addresses of all pages are consecutive in this case. + */ + if (mtr->hem_cfg.is_direct) { + mtr->hem_cfg.root_ba = pages[0]; + return 0; + } + + for (i = 0, mapped_count = 0; i < mtr->hem_cfg.region_count && + mapped_count < page_count; i++) { + r = &mtr->hem_cfg.region[i]; + /* no need to map pages in this region when hopnum is 0 */ + if (!r->hopnum) { + mapped_count += r->count; + continue; + } + + if (r->offset + r->count > page_count) { + ret = -EINVAL; + dev_err(dev, + "failed to check mtr%u count %u + %u > %u.\n", + i, r->offset, r->count, page_count); + return ret; + } + + ret = mtr_map_region(udma_dev, mtr, r, &pages[r->offset], + page_count - mapped_count); + if (ret < 0) { + dev_err(dev, + "failed to map mtr%u offset %u, ret = %d.\n", + i, r->offset, ret); + return ret; + } + mapped_count += ret; + } + ret = 0; + + if (mapped_count < page_count) { + ret = -ENOBUFS; + dev_err(dev, "failed to map mtr pages count: %u < %u.\n", + mapped_count, page_count); + } + + return ret; +} + +int udma_get_kmem_bufs(struct udma_dev *udma_dev, dma_addr_t *bufs, + int buf_cnt, struct udma_buf *buf, + uint32_t page_shift) +{ + uint32_t offset, max_size; + int total = 0; + int i; + + if (page_shift > buf->trunk_shift) { + dev_err(udma_dev->dev, + "failed to check kmem buf shift %u > %u\n", + page_shift, buf->trunk_shift); + return -EINVAL; + } + + offset = 0; + max_size = buf->ntrunks << buf->trunk_shift; + for (i = 0; i < buf_cnt && offset < max_size; i++) { + bufs[total++] = udma_buf_dma_addr(buf, offset); + offset += (1 << page_shift); + } + + return total; +} + +static int mtr_map_bufs(struct udma_dev *udma_dev, struct udma_mtr *mtr, + int page_count, uint32_t page_shift) +{ + struct device *dev = udma_dev->dev; + dma_addr_t *pages; + int npage; + int ret; + + page_shift = need_split_huge_page(&mtr->hem_cfg) ? + UDMA_HW_PAGE_SHIFT : page_shift; + + /* alloc a tmp array to store buffer's dma address */ + pages = kvcalloc(page_count, sizeof(dma_addr_t), GFP_KERNEL); + if (!pages) + return -ENOMEM; + + if (mtr->umem) + npage = udma_get_umem_bufs(udma_dev, pages, page_count, + mtr->umem, page_shift); + else + npage = udma_get_kmem_bufs(udma_dev, pages, page_count, + mtr->kmem, page_shift); + if (npage != page_count) { + dev_err(dev, "failed to get mtr page %d != %d.\n", npage, + page_count); + ret = -ENOBUFS; + goto err_alloc_list; + } + + if (need_split_huge_page(&mtr->hem_cfg) && npage > 1) { + ret = mtr_check_direct_pages(pages, npage, page_shift); + if (ret) { + dev_err(dev, "failed to check %s page: %d / %d.\n", + mtr->umem ? "umtr" : "kmtr", ret, npage); + ret = -ENOBUFS; + goto err_alloc_list; + } + } + + ret = udma_mtr_map(udma_dev, mtr, pages, page_count); + if (ret) + dev_err(dev, "failed to map mtr pages, ret = %d.\n", ret); + +err_alloc_list: + kvfree(pages); + + return ret; +} + +void udma_buf_free(struct udma_dev *udma_dev, struct udma_buf *buf) +{ + struct udma_buf_list *trunks; + uint32_t i; + + if (!buf) + return; + + trunks = buf->trunk_list; + if (trunks) { + buf->trunk_list = NULL; + for (i = 0; i < buf->ntrunks; i++) + dma_free_coherent(udma_dev->dev, 1 << buf->trunk_shift, + trunks[i].buf, trunks[i].map); + + kfree(trunks); + } + + kfree(buf); +} + +static void mtr_free_bufs(struct udma_dev *udma_dev, struct udma_mtr *mtr) +{ + /* release user buffers */ + ubcore_umem_release(mtr->umem); + mtr->umem = NULL; + + /* release kernel buffers */ + udma_buf_free(udma_dev, mtr->kmem); + mtr->kmem = NULL; +} + +static void mtr_free_mtt(struct udma_dev *udma_dev, struct udma_mtr *mtr) +{ + udma_hem_list_release(udma_dev, &mtr->hem_list); +} + +/* + * udma_mtr_create - Create memory translate region. + */ +int udma_mtr_create(struct udma_dev *udma_dev, struct udma_mtr *mtr, + struct udma_buf_attr *buf_attr, uint32_t ba_page_shift, + uint64_t user_addr, bool is_user) +{ + struct device *dev = udma_dev->dev; + uint32_t buf_page_shift = 0; + int buf_page_cnt; + int ret; + + buf_page_cnt = mtr_init_buf_cfg(udma_dev, buf_attr, &mtr->hem_cfg, + &buf_page_shift, + is_user ? user_addr & ~PAGE_MASK : 0); + if (buf_page_cnt < 1 || buf_page_shift < UDMA_HW_PAGE_SHIFT) { + dev_err(dev, "failed to init mtr cfg, count %d shift %u.\n", + buf_page_cnt, buf_page_shift); + return -EINVAL; + } + + ret = mtr_alloc_mtt(udma_dev, mtr, ba_page_shift); + if (ret) { + dev_err(dev, "failed to alloc mtr mtt, ret = %d.\n", ret); + return ret; + } + + /* The caller has its own buffer list and invokes the udma_mtr_map() + * to finish the MTT configuration. + */ + if (buf_attr->mtt_only) { + mtr->umem = NULL; + mtr->kmem = NULL; + return 0; + } + + ret = mtr_alloc_bufs(udma_dev, mtr, buf_attr, user_addr, is_user); + if (ret) { + dev_err(dev, "failed to alloc mtr bufs, ret = %d.\n", ret); + goto err_alloc_mtt; + } + + /* Write buffer's dma address to MTT */ + ret = mtr_map_bufs(udma_dev, mtr, buf_page_cnt, buf_page_shift); + if (ret) + dev_err(dev, "failed to map mtr bufs, ret = %d.\n", ret); + else + return 0; + + mtr_free_bufs(udma_dev, mtr); +err_alloc_mtt: + mtr_free_mtt(udma_dev, mtr); + return ret; +} + +void udma_mtr_destroy(struct udma_dev *udma_dev, struct udma_mtr *mtr) +{ + /* release multi-hop addressing resource */ + udma_hem_list_release(udma_dev, &mtr->hem_list); + + /* free buffers */ + mtr_free_bufs(udma_dev, mtr); +} + +int udma_mtr_find(struct udma_dev *udma_device, struct udma_mtr *mtr, + int offset, uint64_t *mtt_buf, int mtt_max, + uint64_t *base_addr) +{ + int mtt_count, left, start_idx, total; + struct udma_hem_cfg *cfg; + uint32_t npage; + uint64_t *mtts; + uint64_t addr; + + cfg = &mtr->hem_cfg; + total = 0; + + if (!mtt_buf || mtt_max < 1) + goto out; + + /* no mtt memory in direct mode, so just return the buffer address */ + if (cfg->is_direct) { + start_idx = offset >> UDMA_HW_PAGE_SHIFT; + for (mtt_count = 0; (uint32_t)mtt_count < cfg->region_count && + total < mtt_max; mtt_count++) { + npage = cfg->region[mtt_count].offset; + if ((int)npage < start_idx) + continue; + addr = cfg->root_ba + (npage << UDMA_HW_PAGE_SHIFT); + mtt_buf[total] = addr; + + total++; + } + goto out; + } + + start_idx = offset >> cfg->buf_pg_shift; + left = mtt_max; + while (left > 0) { + mtt_count = 0; + mtts = (uint64_t *)udma_hem_list_find_mtt(udma_device, + &mtr->hem_list, + start_idx + total, + &mtt_count, NULL); + if (!mtts || !mtt_count) + goto out; + + npage = min(mtt_count, left); + left -= npage; + for (mtt_count = 0; (uint32_t)mtt_count < npage; mtt_count++) + mtt_buf[total++] = le64_to_cpu(mtts[mtt_count]); + } + +out: + if (base_addr) + *base_addr = cfg->root_ba; + + return total; +} + +void udma_mtr_move(struct udma_mtr *from_mtr, struct udma_mtr *to_mtr) +{ + int i, j; + + *to_mtr = *from_mtr; + udma_hem_list_init(&to_mtr->hem_list); + + list_splice_init(&from_mtr->hem_list.root_bt, + &to_mtr->hem_list.root_bt); + list_splice_init(&from_mtr->hem_list.btm_bt, &to_mtr->hem_list.btm_bt); + + for (i = 0; i < UDMA_MAX_BT_REGION; i++) + for (j = 0; j < UDMA_MAX_BT_LEVEL; j++) + list_splice_init(&from_mtr->hem_list.mid_bt[i][j], + &to_mtr->hem_list.mid_bt[i][j]); +} diff --git a/drivers/ub/hw/hns3/hns3_udma_hem.h b/drivers/ub/hw/hns3/hns3_udma_hem.h new file mode 100644 index 0000000000000000000000000000000000000000..ddf1b9c740755a0ddd4de8f2585e73311e971d10 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_hem.h @@ -0,0 +1,186 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_HEM_H +#define _UDMA_HEM_H + +#include +#include +#include +#include "hns3_udma_device.h" + +#define HEM_HOP_STEP_DIRECT 0xff +#define HEM_INDEX_BUF BIT(0) +#define HEM_INDEX_L0 BIT(1) +#define HEM_INDEX_L1 BIT(2) + +struct udma_hem_index { + uint64_t buf; + uint64_t l0; + uint64_t l1; + uint32_t inited; /* indicate which index is available */ +}; + +enum { + /* UDMA MAP HEM(Hardware Entry Memory) */ + HEM_TYPE_QPC = 0, + HEM_TYPE_MTPT, + HEM_TYPE_CQC, + HEM_TYPE_SRQC, + /* SCC CONTEXT */ + HEM_TYPE_SCCC, + HEM_TYPE_QPC_TIMER, + HEM_TYPE_CQC_TIMER, + HEM_TYPE_GMV, + + /* UDMA UNMAP HEM */ + HEM_TYPE_MTT, + HEM_TYPE_CQE, + HEM_TYPE_SRQWQE, + HEM_TYPE_INDEX, + HEM_TYPE_IRRL, + HEM_TYPE_TRRL, +}; + +#define UDMA_HEM_CHUNK_LEN \ + ((256 - sizeof(struct list_head) - 2 * sizeof(int)) / \ + (sizeof(struct scatterlist) + sizeof(void *))) + +#define check_whether_bt_num_3(type, hop_num) \ + (type < HEM_TYPE_MTT && (hop_num) == 2) + +#define check_whether_bt_num_2(type, hop_num) \ + ((type < HEM_TYPE_MTT && (hop_num) == 1) || \ + (type >= HEM_TYPE_MTT && (hop_num) == 2)) + +#define check_whether_bt_num_1(type, hop_num) \ + (((hop_num) == UDMA_HOP_NUM_0) || \ + (type >= HEM_TYPE_MTT && (hop_num) == 1)) + +#define check_whether_last_step(hop_num, step_idx) \ + (((step_idx) == 0 && (hop_num) == UDMA_HOP_NUM_0) || \ + ((step_idx) == 1 && (hop_num) == 1) || \ + ((step_idx) == 2 && (hop_num) == 2)) + +struct udma_hem_chunk { + struct list_head list; + int npages; + int nsg; + struct scatterlist mem[UDMA_HEM_CHUNK_LEN]; + void *buf[UDMA_HEM_CHUNK_LEN]; +}; + +struct udma_hem { + struct list_head chunk_list; + refcount_t refcount; +}; + +struct udma_hem_iter { + struct udma_hem *hem; + struct udma_hem_chunk *chunk; + int page_idx; +}; + +struct udma_hem_mhop { + uint32_t hop_num; + uint32_t buf_chunk_size; + uint32_t bt_chunk_size; + uint32_t ba_l0_num; + uint32_t l0_idx; /* level 0 base address table index */ + uint32_t l1_idx; /* level 1 base address table index */ + uint32_t l2_idx; /* level 2 base address table index */ +}; + +struct udma_hem_item { + struct list_head list; /* link all hems in the same bt level */ + struct list_head sibling; /* link all hems in last hop for mtt */ + void *addr; + dma_addr_t dma_addr; + size_t count; /* max ba numbers */ + int start; /* start buf offset in this hem */ + int end; /* end buf offset in this hem */ +}; + +/* All HEM itmes are linked in a tree structure */ +struct udma_hem_head { + struct list_head branch[UDMA_MAX_BT_REGION]; + struct list_head root; + struct list_head leaf; +}; + +struct udma_buf *udma_buf_alloc(struct udma_dev *udma_dev, uint32_t size, + uint32_t page_shift, uint32_t flags); +void udma_buf_free(struct udma_dev *udma_dev, struct udma_buf *buf); +int udma_table_get(struct udma_dev *udma_dev, + struct udma_hem_table *table, uint64_t obj); +void udma_table_put(struct udma_dev *udma_dev, + struct udma_hem_table *table, uint64_t obj); +void *udma_table_find(struct udma_dev *udma_dev, + struct udma_hem_table *table, uint64_t obj, + dma_addr_t *dma_handle); +int udma_init_hem_table(struct udma_dev *udma_dev, + struct udma_hem_table *table, uint32_t type, + uint64_t obj_size, uint64_t nobj); +void udma_cleanup_hem_table(struct udma_dev *udma_dev, + struct udma_hem_table *table); +int udma_calc_hem_mhop(struct udma_dev *udma_dev, + struct udma_hem_table *table, uint64_t *obj, + struct udma_hem_mhop *mhop); +bool udma_check_whether_mhop(struct udma_dev *udma_dev, uint32_t type); + +int udma_mtr_find(struct udma_dev *udma_device, struct udma_mtr *mtr, + int offset, uint64_t *mtt_buf, int mtt_max, + uint64_t *base_addr); + +void udma_mtr_move(struct udma_mtr *from_mtr, struct udma_mtr *to_mtr); + +int udma_mtr_map(struct udma_dev *dev, struct udma_mtr *mtr, + dma_addr_t *pages, uint32_t page_count); + +static inline void udma_hem_first(struct udma_hem *hem, + struct udma_hem_iter *iter) +{ + iter->hem = hem; + iter->chunk = list_empty(&hem->chunk_list) ? NULL : + list_entry(hem->chunk_list.next, struct udma_hem_chunk, + list); + iter->page_idx = 0; +} + +static inline int udma_hem_last(struct udma_hem_iter *iter) +{ + return !iter->chunk; +} + +static inline void udma_hem_next(struct udma_hem_iter *iter) +{ + if (++iter->page_idx >= iter->chunk->nsg) { + if (iter->chunk->list.next == &iter->hem->chunk_list) { + iter->chunk = NULL; + return; + } + + iter->chunk = list_entry(iter->chunk->list.next, + struct udma_hem_chunk, list); + iter->page_idx = 0; + } +} + +static inline dma_addr_t udma_hem_addr(struct udma_hem_iter *iter) +{ + return sg_dma_address(&iter->chunk->mem[iter->page_idx]); +} + +#endif /* _UDMA_HEM_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.c b/drivers/ub/hw/hns3/hns3_udma_hw.c new file mode 100644 index 0000000000000000000000000000000000000000..b050ce2fe003ded3e490f481e038e586f0e75d5a --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_hw.c @@ -0,0 +1,2142 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include +#include +#include +#include "urma/ubcore_api.h" +#include "hnae3.h" +#include "hns3_udma_cmd.h" +#include "hns3_udma_hem.h" +#include "hns3_udma_eq.h" +#include "hns3_udma_qp.h" +#include "hns3_udma_dfx.h" +#include "hns3_udma_sysfs.h" + +bool dfx_switch; + +static const struct pci_device_id udma_hw_pci_tbl[] = { + { PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_UDMA_OVER_UBL), + HNAE3_DEV_SUPPORT_UDMA_OVER_UBL_DCB_BITS }, + { PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_UDMA), + HNAE3_DEV_SUPPORT_UDMA_DCB_BITS }, + /* required last entry */ + {} +}; + +static int udma_cmq_query_hw_info(struct udma_dev *udma_dev) +{ + struct udma_query_version *resp; + struct udma_cmq_desc desc; + int ret; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_QUERY_HW_VER, true); + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) + return ret; + + resp = (struct udma_query_version *)desc.data; + + return ret; +} + + +static int udma_query_fw_ver(struct udma_dev *udma_dev) +{ + struct udma_query_fw_info *resp; + struct udma_cmq_desc desc; + int ret; + + udma_cmq_setup_basic_desc(&desc, UDMA_QUERY_FW_VER, true); + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) + return ret; + + resp = (struct udma_query_fw_info *)desc.data; + udma_dev->caps.fw_ver = (uint64_t)(le32_to_cpu(resp->fw_ver)); + + return 0; +} + +static int udma_query_func_id(struct udma_dev *udma_dev) +{ + struct udma_hw_id_query_cmq *resp; + struct udma_cmq_desc desc; + int ret; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_QUERY_HW_ID, true); + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) { + if (desc.retval != CMD_NOT_EXIST) + dev_warn(udma_dev->dev, + "failed to query hw id, ret = %d.\n", ret); + + goto invalid_val; + } + + resp = (struct udma_hw_id_query_cmq *)desc.data; + udma_dev->func_id = (uint16_t)le32_to_cpu(resp->func_id); + return 0; + +invalid_val: + udma_dev->func_id = UDMA_INVALID_ID; + return ret; +} + +static int udma_query_func_info(struct udma_dev *udma_dev) +{ + struct udma_pf_func_info *resp; + struct udma_cmq_desc desc; + int ret; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_QUERY_FUNC_INFO, true); + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) + return ret; + + resp = (struct udma_pf_func_info *)desc.data; + udma_dev->func_num = le32_to_cpu(resp->own_func_num); + udma_dev->cong_algo_tmpl_id = le32_to_cpu(resp->own_mac_id); + + return udma_query_func_id(udma_dev); +} + +static int udma_config_global_param(struct udma_dev *udma_dev) +{ + struct udma_cmq_desc desc; + struct udma_cmq_req *req = (struct udma_cmq_req *)desc.data; + uint32_t clock_cycles_of_1us; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_CFG_GLOBAL_PARAM, + false); + + clock_cycles_of_1us = UDMA_1US_CFG; + + udma_reg_write(req, CFG_GLOBAL_PARAM_1US_CYCLES, clock_cycles_of_1us); + udma_reg_write(req, CFG_GLOBAL_PARAM_UDP_PORT, UDMA_UDP_DPORT); + + return udma_cmq_send(udma_dev, &desc, 1); +} + +static int __udma_set_vf_switch_param(struct udma_dev *udma_dev, + uint32_t vf_id) +{ + struct udma_vf_switch *swt; + struct udma_cmq_desc desc; + int ret; + + swt = (struct udma_vf_switch *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_SWITCH_PARAMETER_CFG, true); + swt->udma_sel |= cpu_to_le32(NIC_ICL_SWITCH_CMD_UDMA_SEL); + udma_set_field(swt->fun_id, VF_SWITCH_DATA_FUN_ID_VF_ID_M, + VF_SWITCH_DATA_FUN_ID_VF_ID_S, vf_id); + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) + return ret; + + desc.flag = + cpu_to_le16(UDMA_CMD_FLAG_NO_INTR | UDMA_CMD_FLAG_IN); + desc.flag &= cpu_to_le16(~UDMA_CMD_FLAG_WR); + udma_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LPBK_S, 1); + udma_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_LCL_LPBK_S, 0); + udma_set_bit(swt->cfg, VF_SWITCH_DATA_CFG_ALW_DST_OVRD_S, 1); + + return udma_cmq_send(udma_dev, &desc, 1); +} + +static int udma_set_vf_switch_param(struct udma_dev *udma_dev) +{ + uint32_t vf_id; + int ret; + + for (vf_id = 0; vf_id < udma_dev->func_num; vf_id++) { + ret = __udma_set_vf_switch_param(udma_dev, vf_id); + if (ret) + return ret; + } + return 0; +} + +static void set_default_jetty_caps(struct udma_dev *dev) +{ + struct udma_caps *caps = &dev->caps; + + caps->num_jfc_shift = UDMA_DEFAULT_MAX_JETTY_X_SHIFT; + caps->num_jfs_shift = UDMA_DEFAULT_MAX_JETTY_X_SHIFT; + caps->num_jfr_shift = UDMA_DEFAULT_MAX_JETTY_X_SHIFT; + caps->num_jetty_shift = UDMA_DEFAULT_MAX_JETTY_X_SHIFT; +} + +static void query_hw_speed(struct udma_dev *udma_dev) +{ + struct udma_port_info_cmq *resp; + struct udma_cmq_desc desc; + int ret; + + resp = (struct udma_port_info_cmq *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_QUERY_PORT_INFO, true); + resp->query_type = UDMA_QUERY_PORT_INFO; + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) { + dev_err(udma_dev->dev, "failed to query speed, ret = %d. set default 100G\n", ret); + udma_dev->caps.speed = SPEED_100G; + return; + } + udma_dev->caps.speed = resp->speed; +} + +static int udma_query_caps(struct udma_dev *udma_dev) +{ + enum udma_opcode_type opcode = UDMA_OPC_QUERY_PF_CAPS_NUM; + struct udma_cmq_desc desc[UDMA_QUERY_PF_CAPS_CMD_NUM]; + struct udma_caps *caps = &udma_dev->caps; + struct udma_query_pf_caps_a *resp_a; + struct udma_query_pf_caps_b *resp_b; + struct udma_query_pf_caps_c *resp_c; + struct udma_query_pf_caps_d *resp_d; + struct udma_query_pf_caps_e *resp_e; + int ctx_hop_num; + int pbl_hop_num; + int ret; + int i; + + for (i = 0; i < UDMA_QUERY_PF_CAPS_CMD_NUM; i++) { + udma_cmq_setup_basic_desc(&desc[i], opcode, true); + if (i < (UDMA_QUERY_PF_CAPS_CMD_NUM - 1)) + desc[i].flag |= cpu_to_le16(UDMA_CMD_FLAG_NEXT); + else + desc[i].flag &= ~cpu_to_le16(UDMA_CMD_FLAG_NEXT); + } + + ret = udma_cmq_send(udma_dev, desc, UDMA_QUERY_PF_CAPS_CMD_NUM); + if (ret) + return ret; + + resp_a = (struct udma_query_pf_caps_a *)desc[0].data; + resp_b = (struct udma_query_pf_caps_b *)desc[1].data; + resp_c = (struct udma_query_pf_caps_c *)desc[2].data; + resp_d = (struct udma_query_pf_caps_d *)desc[3].data; + resp_e = (struct udma_query_pf_caps_e *)desc[4].data; + + caps->local_ca_ack_delay = resp_a->local_ca_ack_delay; + caps->max_sq_sg = le16_to_cpu(resp_a->max_sq_sg); + caps->max_sq_inline = le16_to_cpu(resp_a->max_sq_inline); + caps->max_rq_sg = le16_to_cpu(resp_a->max_rq_sg); + caps->max_rq_sg = roundup_pow_of_two(caps->max_rq_sg); + caps->max_extend_sg = le32_to_cpu(resp_a->max_extend_sg); + caps->num_qpc_timer = le16_to_cpu(resp_a->num_qpc_timer); + caps->max_srq_sges = le16_to_cpu(resp_a->max_srq_sges); + /* reserved for UM header */ + caps->max_srq_sges = roundup_pow_of_two(caps->max_srq_sges) - 1; + caps->num_aeq_vectors = resp_a->num_aeq_vectors; + caps->num_other_vectors = resp_a->num_other_vectors; + caps->max_sq_desc_sz = resp_a->max_sq_desc_sz; + caps->max_rq_desc_sz = resp_a->max_rq_desc_sz; + caps->max_srq_desc_sz = resp_a->max_srq_desc_sz; + caps->cqe_sz = resp_a->cqe_sz; + + caps->mtpt_entry_sz = resp_b->mtpt_entry_sz; + caps->irrl_entry_sz = resp_b->irrl_entry_sz; + caps->trrl_entry_sz = resp_b->trrl_entry_sz; + caps->cqc_entry_sz = resp_b->cqc_entry_sz; + caps->srqc_entry_sz = resp_b->srqc_entry_sz; + caps->idx_entry_sz = resp_b->idx_entry_sz; + caps->scc_ctx_sz = resp_b->sccc_sz; + caps->max_mtu = (enum ubcore_mtu)resp_b->max_mtu; + caps->qpc_sz = le16_to_cpu(resp_b->qpc_sz); + caps->min_cqes = resp_b->min_cqes; + caps->min_wqes = resp_b->min_wqes; + caps->page_size_cap = le32_to_cpu(resp_b->page_size_cap); + caps->pkey_table_len[0] = resp_b->pkey_table_len; + caps->phy_num_uars = resp_b->phy_num_uars; + ctx_hop_num = resp_b->ctx_hop_num; + pbl_hop_num = resp_b->pbl_hop_num; + + caps->num_pds = 1 << udma_get_field(resp_c->cap_flags_num_pds, + QUERY_PF_CAPS_C_NUM_PDS_M, + QUERY_PF_CAPS_C_NUM_PDS_S); + caps->flags = udma_get_field(resp_c->cap_flags_num_pds, + QUERY_PF_CAPS_C_CAP_FLAGS_M, + QUERY_PF_CAPS_C_CAP_FLAGS_S); + caps->flags |= le16_to_cpu(resp_d->cap_flags_ex) << + UDMA_CAP_FLAGS_EX_SHIFT; + + caps->num_cqs = 1 << udma_get_field(resp_c->max_gid_num_cqs, + QUERY_PF_CAPS_C_NUM_CQS_M, + QUERY_PF_CAPS_C_NUM_CQS_S); + caps->gid_table_len[0] = udma_get_field(resp_c->max_gid_num_cqs, + QUERY_PF_CAPS_C_MAX_GID_M, + QUERY_PF_CAPS_C_MAX_GID_S); + + caps->max_cqes = 1 << udma_get_field(resp_c->cq_depth, + QUERY_PF_CAPS_C_CQ_DEPTH_M, + QUERY_PF_CAPS_C_CQ_DEPTH_S); + caps->num_mtpts = 1 << udma_get_field(resp_c->num_mrws, + QUERY_PF_CAPS_C_NUM_MRWS_M, + QUERY_PF_CAPS_C_NUM_MRWS_S); + caps->num_qps = 1 << udma_get_field(resp_c->ord_num_qps, + QUERY_PF_CAPS_C_NUM_QPS_M, + QUERY_PF_CAPS_C_NUM_QPS_S); + caps->num_qps_shift = udma_get_field(resp_c->ord_num_qps, + QUERY_PF_CAPS_C_NUM_QPS_M, + QUERY_PF_CAPS_C_NUM_QPS_S); + caps->max_qp_init_rdma = udma_get_field(resp_c->ord_num_qps, + QUERY_PF_CAPS_C_MAX_ORD_M, + QUERY_PF_CAPS_C_MAX_ORD_S); + caps->max_qp_dest_rdma = caps->max_qp_init_rdma; + caps->max_wqes = 1 << le16_to_cpu(resp_c->sq_depth); + caps->num_srqs = 1 << udma_get_field(resp_d->wq_hop_num_max_srqs, + QUERY_PF_CAPS_D_NUM_SRQS_M, + QUERY_PF_CAPS_D_NUM_SRQS_S); + caps->cong_type = udma_get_field(resp_d->wq_hop_num_max_srqs, + QUERY_PF_CAPS_D_CONG_TYPE_M, + QUERY_PF_CAPS_D_CONG_TYPE_S); + caps->max_srq_wrs = 1 << le16_to_cpu(resp_d->srq_depth); + + caps->ceqe_depth = 1 << udma_get_field(resp_d->num_ceqs_ceq_depth, + QUERY_PF_CAPS_D_CEQ_DEPTH_M, + QUERY_PF_CAPS_D_CEQ_DEPTH_S); + caps->num_comp_vectors = udma_get_field(resp_d->num_ceqs_ceq_depth, + QUERY_PF_CAPS_D_NUM_CEQS_M, + QUERY_PF_CAPS_D_NUM_CEQS_S); + + caps->aeqe_depth = 1 << udma_get_field(resp_d->arm_st_aeq_depth, + QUERY_PF_CAPS_D_AEQ_DEPTH_M, + QUERY_PF_CAPS_D_AEQ_DEPTH_S); + caps->default_aeq_arm_st = udma_get_field(resp_d->arm_st_aeq_depth, + QUERY_PF_CAPS_D_AEQ_ARM_ST_M, + QUERY_PF_CAPS_D_AEQ_ARM_ST_S); + caps->default_ceq_arm_st = udma_get_field(resp_d->arm_st_aeq_depth, + QUERY_PF_CAPS_D_CEQ_ARM_ST_M, + QUERY_PF_CAPS_D_CEQ_ARM_ST_S); + caps->reserved_pds = udma_get_field(resp_d->num_uars_rsv_pds, + QUERY_PF_CAPS_D_RSV_PDS_M, + QUERY_PF_CAPS_D_RSV_PDS_S); + caps->num_uars = 1 << udma_get_field(resp_d->num_uars_rsv_pds, + QUERY_PF_CAPS_D_NUM_UARS_M, + QUERY_PF_CAPS_D_NUM_UARS_S); + caps->reserved_qps = udma_get_field(resp_d->rsv_uars_rsv_qps, + QUERY_PF_CAPS_D_RSV_QPS_M, + QUERY_PF_CAPS_D_RSV_QPS_S); + caps->reserved_uars = udma_get_field(resp_d->rsv_uars_rsv_qps, + QUERY_PF_CAPS_D_RSV_UARS_M, + QUERY_PF_CAPS_D_RSV_UARS_S); + caps->reserved_mrws = udma_get_field(resp_e->chunk_size_shift_rsv_mrws, + QUERY_PF_CAPS_E_RSV_MRWS_M, + QUERY_PF_CAPS_E_RSV_MRWS_S); + caps->chunk_sz = 1 << udma_get_field(resp_e->chunk_size_shift_rsv_mrws, + QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_M, + QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_S); + caps->reserved_cqs = udma_get_field(resp_e->rsv_cqs, + QUERY_PF_CAPS_E_RSV_CQS_M, + QUERY_PF_CAPS_E_RSV_CQS_S); + caps->reserved_srqs = udma_get_field(resp_e->rsv_srqs, + QUERY_PF_CAPS_E_RSV_SRQS_M, + QUERY_PF_CAPS_E_RSV_SRQS_S); + caps->reserved_lkey = udma_get_field(resp_e->rsv_lkey, + QUERY_PF_CAPS_E_RSV_LKEYS_M, + QUERY_PF_CAPS_E_RSV_LKEYS_S); + caps->default_ceq_max_cnt = le16_to_cpu(resp_e->ceq_max_cnt); + caps->default_ceq_period = le16_to_cpu(resp_e->ceq_period); + caps->default_aeq_max_cnt = le16_to_cpu(resp_e->aeq_max_cnt); + caps->default_aeq_period = le16_to_cpu(resp_e->aeq_period); + + caps->qpc_hop_num = ctx_hop_num; + caps->sccc_hop_num = ctx_hop_num; + caps->srqc_hop_num = ctx_hop_num; + caps->cqc_hop_num = ctx_hop_num; + caps->mpt_hop_num = ctx_hop_num; + caps->mtt_hop_num = pbl_hop_num; + caps->cqe_hop_num = pbl_hop_num; + caps->srqwqe_hop_num = pbl_hop_num; + caps->idx_hop_num = pbl_hop_num; + caps->wqe_sq_hop_num = udma_get_field(resp_d->wq_hop_num_max_srqs, + QUERY_PF_CAPS_D_SQWQE_HOP_NUM_M, + QUERY_PF_CAPS_D_SQWQE_HOP_NUM_S); + caps->wqe_sge_hop_num = udma_get_field(resp_d->wq_hop_num_max_srqs, + QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_M, + QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_S); + caps->wqe_rq_hop_num = udma_get_field(resp_d->wq_hop_num_max_srqs, + QUERY_PF_CAPS_D_RQWQE_HOP_NUM_M, + QUERY_PF_CAPS_D_RQWQE_HOP_NUM_S); + + set_default_jetty_caps(udma_dev); + query_hw_speed(udma_dev); + + return 0; +} + +static int load_func_res_caps(struct udma_dev *udma_dev) +{ + struct udma_cmq_desc desc[2]; + struct udma_cmq_req *r_a = (struct udma_cmq_req *)desc[0].data; + struct udma_cmq_req *r_b = (struct udma_cmq_req *)desc[1].data; + struct udma_caps *caps = &udma_dev->caps; + enum udma_opcode_type opcode; + uint32_t func_num; + int ret; + + opcode = UDMA_OPC_QUERY_PF_RES; + func_num = udma_dev->func_num; + + udma_cmq_setup_basic_desc(&desc[0], opcode, true); + desc[0].flag |= cpu_to_le16(UDMA_CMD_FLAG_NEXT); + udma_cmq_setup_basic_desc(&desc[1], opcode, true); + + ret = udma_cmq_send(udma_dev, desc, 2); + if (ret) + return ret; + + caps->qpc_bt_num = udma_reg_read(r_a, FUNC_RES_A_QPC_BT_NUM) / func_num; + caps->srqc_bt_num = + udma_reg_read(r_a, FUNC_RES_A_SRQC_BT_NUM) / func_num; + caps->cqc_bt_num = udma_reg_read(r_a, FUNC_RES_A_CQC_BT_NUM) / func_num; + caps->mpt_bt_num = udma_reg_read(r_a, FUNC_RES_A_MPT_BT_NUM) / func_num; + caps->eqc_bt_num = udma_reg_read(r_a, FUNC_RES_A_EQC_BT_NUM) / func_num; + caps->smac_bt_num = udma_reg_read(r_b, FUNC_RES_B_SMAC_NUM) / func_num; + caps->sgid_bt_num = udma_reg_read(r_b, FUNC_RES_B_SGID_NUM) / func_num; + caps->sccc_bt_num = + udma_reg_read(r_b, FUNC_RES_B_SCCC_BT_NUM) / func_num; + + + caps->sl_num = + udma_reg_read(r_b, FUNC_RES_B_QID_NUM) / func_num; + caps->gmv_bt_num = + udma_reg_read(r_b, FUNC_RES_B_GMV_BT_NUM) / func_num; + + return 0; +} + +static void setup_default_ext_caps(struct udma_dev *udma_dev) +{ + struct udma_caps *caps = &udma_dev->caps; + + caps->num_pi_qps = caps->num_qps; + caps->llm_ba_idx = 0; + caps->llm_ba_num = UDMA_EXT_LLM_MAX_DEPTH; + +} + +static int load_ext_cfg_caps(struct udma_dev *udma_dev) +{ + struct udma_cmq_desc desc; + struct udma_cmq_req *req = (struct udma_cmq_req *)desc.data; + struct udma_caps *caps = &udma_dev->caps; + uint32_t func_num, qp_num; + int ret; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_EXT_CFG, true); + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) + return ret; + + func_num = max_t(uint32_t, 1, udma_dev->func_num); + qp_num = udma_reg_read(req, EXT_CFG_QP_PI_NUM) / func_num; + caps->num_pi_qps = round_down(qp_num, UDMA_QP_BANK_NUM); + + qp_num = udma_reg_read(req, EXT_CFG_QP_NUM) / func_num; + caps->num_qps = round_down(qp_num, UDMA_QP_BANK_NUM); + caps->num_qps_shift = ilog2(caps->num_qps); + + /* The extend doorbell memory on the PF is shared by all its VFs. */ + caps->llm_ba_idx = udma_reg_read(req, EXT_CFG_LLM_IDX); + caps->llm_ba_num = udma_reg_read(req, EXT_CFG_LLM_NUM); + + return 0; +} + +static int query_func_resource_caps(struct udma_dev *udma_dev) +{ + struct device *dev = udma_dev->dev; + int ret; + + ret = load_func_res_caps(udma_dev); + if (ret) { + dev_err(dev, "failed to load res caps, ret = %d (pf).\n", + ret); + return ret; + } + + setup_default_ext_caps(udma_dev); + + ret = load_ext_cfg_caps(udma_dev); + if (ret) + dev_err(dev, "failed to load ext cfg, ret = %d (pf).\n", + ret); + return ret; +} + +static int load_pf_timer_res_caps(struct udma_dev *udma_dev) +{ + struct udma_cmq_desc desc; + struct udma_cmq_req *req = (struct udma_cmq_req *)desc.data; + struct udma_caps *caps = &udma_dev->caps; + int ret; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_QUERY_PF_TIMER_RES, + true); + + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) + return ret; + + caps->qpc_timer_bt_num = udma_reg_read(req, PF_TIMER_RES_QPC_ITEM_NUM); + caps->cqc_timer_bt_num = udma_reg_read(req, PF_TIMER_RES_CQC_ITEM_NUM); + + return 0; +} + +static int query_func_oor_caps(struct udma_dev *udma_dev) +{ + struct udma_caps *caps = &udma_dev->caps; + struct udma_query_oor_cmq *resp; + struct udma_cmq_desc desc; + int ret = 0; + + udma_cmq_setup_basic_desc(&desc, UDMA_QUERY_OOR_CAPS, true); + + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) { + dev_err(udma_dev->dev, + "failed to func oor caps, ret = %d.\n", ret); + return ret; + } + + resp = (struct udma_query_oor_cmq *)desc.data; + + caps->oor_en = resp->oor_en; + caps->reorder_cq_buffer_en = resp->reorder_cq_buffer_en; + caps->reorder_cap = resp->reorder_cap; + caps->reorder_cq_shift = resp->reorder_cq_shift; + caps->onflight_size = resp->on_flight_size; + caps->dynamic_ack_timeout = resp->dynamic_ack_timeout; + + return ret; +} + +static int udma_query_pf_resource(struct udma_dev *udma_dev) +{ + struct device *dev = udma_dev->dev; + int ret; + + ret = query_func_resource_caps(udma_dev); + if (ret) + return ret; + + ret = load_pf_timer_res_caps(udma_dev); + if (ret) { + dev_err(dev, "failed to load pf timer resource, ret = %d.\n", + ret); + return ret; + } + + if (udma_dev->caps.flags & UDMA_CAP_FLAG_OOR) + ret = query_func_oor_caps(udma_dev); + + return ret; +} + +static void calc_pg_sz(uint32_t obj_num, uint32_t obj_size, uint32_t hop_num, + uint32_t ctx_bt_num, uint32_t *buf_page_size, + uint32_t *bt_page_size, uint32_t hem_type) +{ + uint64_t buf_chunk_size = PAGE_SIZE; + uint64_t bt_chunk_size = PAGE_SIZE; + uint64_t obj_per_chunk_default; + uint64_t obj_per_chunk; + + obj_per_chunk_default = buf_chunk_size / obj_size; + *buf_page_size = 0; + *bt_page_size = 0; + + switch (hop_num) { + case 3: + obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) * + (bt_chunk_size / BA_BYTE_LEN) * + (bt_chunk_size / BA_BYTE_LEN) * + obj_per_chunk_default; + break; + case 2: + obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) * + (bt_chunk_size / BA_BYTE_LEN) * + obj_per_chunk_default; + break; + case 1: + obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) * + obj_per_chunk_default; + break; + case UDMA_HOP_NUM_0: + obj_per_chunk = ctx_bt_num * obj_per_chunk_default; + break; + default: + pr_err("table %u not support hop_num = %u!\n", hem_type, + hop_num); + return; + } + + if (hem_type >= HEM_TYPE_MTT) + *bt_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk)); + else + *buf_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk)); +} + +static void set_hem_page_size(struct udma_dev *udma_dev) +{ + struct udma_caps *caps = &udma_dev->caps; + + /* EQ */ + caps->eqe_ba_pg_sz = 0; + caps->eqe_buf_pg_sz = 0; + + /* Link Table */ + caps->llm_buf_pg_sz = 0; + + /* MR */ + caps->mpt_ba_pg_sz = 0; + caps->mpt_buf_pg_sz = 0; + caps->pbl_ba_pg_sz = UDMA_BA_PG_SZ_SUPPORTED_16K; + caps->pbl_buf_pg_sz = 0; + calc_pg_sz(caps->num_mtpts, caps->mtpt_entry_sz, caps->mpt_hop_num, + caps->mpt_bt_num, &caps->mpt_buf_pg_sz, &caps->mpt_ba_pg_sz, + HEM_TYPE_MTPT); + + /* QP */ + caps->qpc_ba_pg_sz = 0; + caps->qpc_buf_pg_sz = 0; + caps->qpc_timer_ba_pg_sz = 0; + caps->qpc_timer_buf_pg_sz = 0; + caps->sccc_ba_pg_sz = 0; + caps->sccc_buf_pg_sz = 0; + caps->mtt_ba_pg_sz = 0; + caps->mtt_buf_pg_sz = 0; + calc_pg_sz(caps->num_qps, caps->qpc_sz, caps->qpc_hop_num, + caps->qpc_bt_num, &caps->qpc_buf_pg_sz, &caps->qpc_ba_pg_sz, + HEM_TYPE_QPC); + + if (caps->flags & UDMA_CAP_FLAG_QP_FLOW_CTRL) + calc_pg_sz(caps->num_qps, caps->scc_ctx_sz, caps->sccc_hop_num, + caps->sccc_bt_num, &caps->sccc_buf_pg_sz, + &caps->sccc_ba_pg_sz, HEM_TYPE_SCCC); + + /* CQ */ + caps->cqc_ba_pg_sz = 0; + caps->cqc_buf_pg_sz = 0; + caps->cqc_timer_ba_pg_sz = 0; + caps->cqc_timer_buf_pg_sz = 0; + caps->cqe_ba_pg_sz = UDMA_BA_PG_SZ_SUPPORTED_256K; + caps->cqe_buf_pg_sz = 0; + calc_pg_sz(caps->num_cqs, caps->cqc_entry_sz, caps->cqc_hop_num, + caps->cqc_bt_num, &caps->cqc_buf_pg_sz, &caps->cqc_ba_pg_sz, + HEM_TYPE_CQC); + calc_pg_sz(caps->max_cqes, caps->cqe_sz, caps->cqe_hop_num, + 1, &caps->cqe_buf_pg_sz, &caps->cqe_ba_pg_sz, HEM_TYPE_CQE); + + /* SRQ */ + if (caps->flags & UDMA_CAP_FLAG_SRQ) { + caps->srqc_ba_pg_sz = 0; + caps->srqc_buf_pg_sz = 0; + caps->srqwqe_ba_pg_sz = 0; + caps->srqwqe_buf_pg_sz = 0; + caps->idx_ba_pg_sz = 0; + caps->idx_buf_pg_sz = 0; + calc_pg_sz(caps->num_srqs, caps->srqc_entry_sz, + caps->srqc_hop_num, caps->srqc_bt_num, + &caps->srqc_buf_pg_sz, &caps->srqc_ba_pg_sz, + HEM_TYPE_SRQC); + calc_pg_sz(caps->num_srqwqe_segs, caps->mtt_entry_sz, + caps->srqwqe_hop_num, 1, &caps->srqwqe_buf_pg_sz, + &caps->srqwqe_ba_pg_sz, HEM_TYPE_SRQWQE); + calc_pg_sz(caps->num_idx_segs, caps->idx_entry_sz, + caps->idx_hop_num, 1, &caps->idx_buf_pg_sz, + &caps->idx_ba_pg_sz, HEM_TYPE_INDEX); + } + + /* GMV */ + caps->gmv_ba_pg_sz = 0; + caps->gmv_buf_pg_sz = 0; +} + +/* Apply all loaded caps before setting to hardware */ +static void apply_func_caps(struct udma_dev *udma_dev) +{ + struct udma_caps *caps = &udma_dev->caps; + struct udma_priv *priv = (struct udma_priv *)udma_dev->priv; + + caps->qpc_timer_entry_sz = UDMA_QPC_TIMER_ENTRY_SZ; + caps->cqc_timer_entry_sz = UDMA_CQC_TIMER_ENTRY_SZ; + caps->mtt_entry_sz = UDMA_MTT_ENTRY_SZ; + caps->eqe_hop_num = UDMA_EQE_HOP_NUM; + caps->pbl_hop_num = UDMA_PBL_HOP_NUM; + caps->qpc_timer_hop_num = UDMA_HOP_NUM_0; + caps->cqc_timer_hop_num = UDMA_HOP_NUM_0; + caps->ceqe_size = UDMA_EQE_SIZE; + caps->aeqe_size = UDMA_EQE_SIZE; + caps->num_mtt_segs = UDMA_MAX_MTT_SEGS; + caps->num_srqwqe_segs = UDMA_MAX_SRQWQE_SEGS; + caps->num_idx_segs = UDMA_MAX_IDX_SEGS; + + /* num_vector = comp_vector + aeq_vector + abn_vector */ + if (!caps->num_comp_vectors) { + caps->num_comp_vectors = + min_t(uint32_t, caps->eqc_bt_num - 1, + (uint32_t)priv->handle->udmainfo.num_vectors - + UDMA_FUNC_IRQ_RSV); + } + caps->qpc_sz = UDMA_QPC_SZ; + caps->cqe_sz = UDMA_CQE_SZ; + caps->scc_ctx_sz = UDMA_SCCC_SZ; + + /* The following caps are not in ncl config */ + caps->gmv_entry_sz = UDMA_GMV_ENTRY_SZ; + + caps->gmv_hop_num = UDMA_HOP_NUM_0; + caps->gid_table_len[0] = caps->gmv_bt_num * + (UDMA_PAGE_SIZE / caps->gmv_entry_sz); + + caps->gmv_entry_num = caps->gmv_bt_num * (PAGE_SIZE / + caps->gmv_entry_sz); + + set_hem_page_size(udma_dev); +} + +static int config_vf_ext_resource(struct udma_dev *udma_dev, uint32_t vf_id) +{ + struct udma_cmq_desc desc; + struct udma_cmq_req *req = (struct udma_cmq_req *)desc.data; + struct udma_caps *caps = &udma_dev->caps; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_EXT_CFG, false); + + udma_reg_write(req, EXT_CFG_VF_ID, vf_id); + + udma_reg_write(req, EXT_CFG_QP_PI_NUM, caps->num_pi_qps); + udma_reg_write(req, EXT_CFG_QP_PI_INDEX, vf_id * caps->num_pi_qps); + udma_reg_write(req, EXT_CFG_QP_NUM, caps->num_qps); + udma_reg_write(req, EXT_CFG_QP_INDEX, vf_id * caps->num_qps); + + return udma_cmq_send(udma_dev, &desc, 1); +} + +static int config_vf_hem_resource(struct udma_dev *udma_dev, int vf_id) +{ + struct udma_cmq_desc desc[2]; + struct udma_cmq_req *r_a = (struct udma_cmq_req *)desc[0].data; + struct udma_cmq_req *r_b = (struct udma_cmq_req *)desc[1].data; + struct udma_caps *caps = &udma_dev->caps; + + udma_cmq_setup_basic_desc(&desc[0], UDMA_OPC_ALLOC_VF_RES, false); + desc[0].flag |= cpu_to_le16(UDMA_CMD_FLAG_NEXT); + udma_cmq_setup_basic_desc(&desc[1], UDMA_OPC_ALLOC_VF_RES, false); + + udma_reg_write(r_a, FUNC_RES_A_VF_ID, vf_id); + + udma_reg_write(r_a, FUNC_RES_A_QPC_BT_NUM, caps->qpc_bt_num); + udma_reg_write(r_a, FUNC_RES_A_QPC_BT_INDEX, vf_id * caps->qpc_bt_num); + udma_reg_write(r_a, FUNC_RES_A_SRQC_BT_NUM, caps->srqc_bt_num); + udma_reg_write(r_a, FUNC_RES_A_SRQC_BT_INDEX, vf_id * caps->srqc_bt_num); + udma_reg_write(r_a, FUNC_RES_A_CQC_BT_NUM, caps->cqc_bt_num); + udma_reg_write(r_a, FUNC_RES_A_CQC_BT_INDEX, vf_id * caps->cqc_bt_num); + udma_reg_write(r_a, FUNC_RES_A_MPT_BT_NUM, caps->mpt_bt_num); + udma_reg_write(r_a, FUNC_RES_A_MPT_BT_INDEX, vf_id * caps->mpt_bt_num); + udma_reg_write(r_a, FUNC_RES_A_EQC_BT_NUM, caps->eqc_bt_num); + udma_reg_write(r_a, FUNC_RES_A_EQC_BT_INDEX, vf_id * caps->eqc_bt_num); + udma_reg_write(r_b, FUNC_RES_V_QID_NUM, caps->sl_num); + udma_reg_write(r_b, FUNC_RES_B_QID_INDEX, vf_id * caps->sl_num); + udma_reg_write(r_b, FUNC_RES_B_SCCC_BT_NUM, caps->sccc_bt_num); + udma_reg_write(r_b, FUNC_RES_B_SCCC_BT_INDEX, vf_id * caps->sccc_bt_num); + udma_reg_write(r_b, FUNC_RES_V_GMV_BT_NUM, caps->gmv_bt_num); + udma_reg_write(r_b, FUNC_RES_B_GMV_BT_INDEX, + vf_id * caps->gmv_bt_num); + + return udma_cmq_send(udma_dev, desc, 2); +} + +static int udma_alloc_vf_resource(struct udma_dev *udma_dev) +{ + uint32_t vf_id; + int ret; + + for (vf_id = 0; vf_id < udma_dev->func_num; vf_id++) { + ret = config_vf_hem_resource(udma_dev, vf_id); + if (ret) { + dev_err(udma_dev->dev, + "failed to config vf-%u hem res, ret = %d.\n", + vf_id, ret); + return ret; + } + + ret = config_vf_ext_resource(udma_dev, vf_id); + if (ret) { + dev_err(udma_dev->dev, + "failed to config vf-%u ext res, ret = %d.\n", + vf_id, ret); + return ret; + } + } + + return 0; +} + +static int udma_set_bt(struct udma_dev *udma_dev) +{ + struct udma_cmq_desc desc; + struct udma_cmq_req *req = (struct udma_cmq_req *)desc.data; + struct udma_caps *caps = &udma_dev->caps; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_CFG_BT_ATTR, false); + + udma_reg_write(req, CFG_BT_QPC_BA_PGSZ, + caps->qpc_ba_pg_sz + PG_SHIFT_OFFSET); + udma_reg_write(req, CFG_BT_QPC_BUF_PGSZ, + caps->qpc_buf_pg_sz + PG_SHIFT_OFFSET); + udma_reg_write(req, CFG_BT_QPC_HOPNUM, + to_udma_hem_hopnum(caps->qpc_hop_num, caps->num_qps)); + + udma_reg_write(req, CFG_BT_SRQC_BA_PGSZ, + caps->srqc_ba_pg_sz + PG_SHIFT_OFFSET); + udma_reg_write(req, CFG_BT_SRQC_BUF_PGSZ, + caps->srqc_buf_pg_sz + PG_SHIFT_OFFSET); + udma_reg_write(req, CFG_BT_SRQC_HOPNUM, + to_udma_hem_hopnum(caps->srqc_hop_num, caps->num_srqs)); + + udma_reg_write(req, CFG_BT_CQC_BA_PGSZ, + caps->cqc_ba_pg_sz + PG_SHIFT_OFFSET); + udma_reg_write(req, CFG_BT_CQC_BUF_PGSZ, + caps->cqc_buf_pg_sz + PG_SHIFT_OFFSET); + udma_reg_write(req, CFG_BT_CQC_HOPNUM, + to_udma_hem_hopnum(caps->cqc_hop_num, caps->num_cqs)); + + udma_reg_write(req, CFG_BT_MPT_BA_PGSZ, + caps->mpt_ba_pg_sz + PG_SHIFT_OFFSET); + udma_reg_write(req, CFG_BT_MPT_BUF_PGSZ, + caps->mpt_buf_pg_sz + PG_SHIFT_OFFSET); + udma_reg_write(req, CFG_BT_MPT_HOPNUM, + to_udma_hem_hopnum(caps->mpt_hop_num, caps->num_mtpts)); + + udma_reg_write(req, CFG_BT_SCCC_BA_PGSZ, + caps->sccc_ba_pg_sz + PG_SHIFT_OFFSET); + udma_reg_write(req, CFG_BT_SCCC_BUF_PGSZ, + caps->sccc_buf_pg_sz + PG_SHIFT_OFFSET); + udma_reg_write(req, CFG_BT_SCCC_HOPNUM, + to_udma_hem_hopnum(caps->sccc_hop_num, caps->num_qps)); + + return udma_cmq_send(udma_dev, &desc, 1); +} + +static int config_hem_entry_size(struct udma_dev *udma_dev, uint32_t type, + uint32_t val) +{ + struct udma_cmq_desc desc; + struct udma_cmq_req *req = (struct udma_cmq_req *)desc.data; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_CFG_ENTRY_SIZE, + false); + + udma_reg_write(req, CFG_HEM_ENTRY_SIZE_TYPE, type); + udma_reg_write(req, CFG_HEM_ENTRY_SIZE_VALUE, val); + + return udma_cmq_send(udma_dev, &desc, 1); +} + +static int udma_config_entry_size(struct udma_dev *udma_dev) +{ + struct udma_caps *caps = &udma_dev->caps; + int ret; + + ret = config_hem_entry_size(udma_dev, UDMA_CFG_QPC_SIZE, + caps->qpc_sz); + if (ret) { + dev_err(udma_dev->dev, + "failed to cfg qpc sz, ret = %d.\n", ret); + return ret; + } + + ret = config_hem_entry_size(udma_dev, UDMA_CFG_SCCC_SIZE, + caps->scc_ctx_sz); + if (ret) + dev_err(udma_dev->dev, + "failed to cfg sccc sz, ret = %d.\n", ret); + + return ret; +} + +static int udma_pf_profile(struct udma_dev *udma_dev) +{ + struct device *dev = udma_dev->dev; + int ret; + + ret = udma_query_func_info(udma_dev); + if (ret) { + dev_err(dev, "failed to query func info, ret = %d.\n", ret); + return ret; + } + + ret = udma_config_global_param(udma_dev); + if (ret) { + dev_err(dev, "failed to config global param, ret = %d.\n", ret); + return ret; + } + + ret = udma_set_vf_switch_param(udma_dev); + if (ret) { + dev_err(dev, "failed to set switch param, ret = %d.\n", ret); + return ret; + } + + ret = udma_query_caps(udma_dev); + if (ret) { + dev_err(dev, "failed to query caps, ret = %d.\n", ret); + return ret; + } + + ret = udma_query_pf_resource(udma_dev); + if (ret) { + dev_err(dev, "failed to query pf resource, ret = %d.\n", ret); + return ret; + } + + apply_func_caps(udma_dev); + ret = udma_alloc_vf_resource(udma_dev); + if (ret) { + dev_err(dev, "failed to alloc vf resource, ret = %d.\n", ret); + return ret; + } + + ret = udma_set_bt(udma_dev); + if (ret) { + dev_err(dev, "failed to config BA table, ret = %d.\n", ret); + return ret; + } + + /* Configure the size of QPC, SCCC, etc. */ + return udma_config_entry_size(udma_dev); +} + +static int udma_profile(struct udma_dev *udma_dev) +{ + struct device *dev = udma_dev->dev; + int ret; + + ret = udma_cmq_query_hw_info(udma_dev); + if (ret) { + dev_err(dev, "failed to query hardware info, ret = %d.\n", ret); + return ret; + } + + ret = udma_query_fw_ver(udma_dev); + if (ret) { + dev_err(dev, "failed to query firmware info, ret = %d.\n", ret); + return ret; + } + + udma_dev->sys_image_guid = be64_to_cpu(udma_dev->ub_dev.attr.guid); + + return udma_pf_profile(udma_dev); +} + +static int udma_alloc_cmq_desc(struct udma_dev *udma_dev, + struct udma_cmq_ring *ring) +{ + int size = ring->desc_num * sizeof(struct udma_cmq_desc); + + ring->desc = kzalloc(size, GFP_KERNEL); + if (!ring->desc) + return -ENOMEM; + + ring->desc_dma_addr = dma_map_single(udma_dev->dev, ring->desc, size, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(udma_dev->dev, ring->desc_dma_addr)) { + ring->desc_dma_addr = 0; + kfree(ring->desc); + ring->desc = NULL; + + dev_err_ratelimited(udma_dev->dev, + "failed to map cmq desc addr.\n"); + return -ENOMEM; + } + + return 0; +} + +static void udma_free_cmq_desc(struct udma_dev *udma_dev, + struct udma_cmq_ring *ring) +{ + dma_unmap_single(udma_dev->dev, ring->desc_dma_addr, + ring->desc_num * sizeof(struct udma_cmq_desc), + DMA_BIDIRECTIONAL); + + ring->desc_dma_addr = 0; + kfree(ring->desc); + ring->desc = NULL; +} + +static int init_csq(struct udma_dev *udma_dev, + struct udma_cmq_ring *csq) +{ + dma_addr_t dma; + int ret; + + csq->desc_num = UDMA_CMD_CSQ_DESC_NUM; + mutex_init(&csq->lock); + csq->flag = TYPE_CSQ; + csq->head = 0; + + ret = udma_alloc_cmq_desc(udma_dev, csq); + if (ret) + return ret; + + dma = csq->desc_dma_addr; + ub_write(udma_dev, UDMA_TX_CMQ_BASEADDR_L_REG, lower_32_bits(dma)); + ub_write(udma_dev, UDMA_TX_CMQ_BASEADDR_H_REG, upper_32_bits(dma)); + ub_write(udma_dev, UDMA_TX_CMQ_DEPTH_REG, + (uint32_t)csq->desc_num >> UDMA_CMQ_DESC_NUM_S); + + /* Make sure to write CI first and then PI */ + ub_write(udma_dev, UDMA_TX_CMQ_CI_REG, 0); + ub_write(udma_dev, UDMA_TX_CMQ_PI_REG, 0); + + return 0; +} + +static int udma_cmq_init(struct udma_dev *udma_dev) +{ + struct udma_priv *priv = (struct udma_priv *)udma_dev->priv; + int ret; + + priv->cmq.tx_timeout = UDMA_CMQ_TX_TIMEOUT; + + ret = init_csq(udma_dev, &priv->cmq.csq); + if (ret) + dev_err(udma_dev->dev, "failed to init CSQ, ret = %d.\n", ret); + + return ret; +} + +static void udma_cmq_exit(struct udma_dev *udma_dev) +{ + struct udma_priv *priv = (struct udma_priv *)udma_dev->priv; + + udma_free_cmq_desc(udma_dev, &priv->cmq.csq); +} + +static int config_gmv_table(struct udma_dev *udma_dev, union ubcore_eid eid) +{ + uint32_t sgid_type = SGID_TYPE_IPV4; + struct udma_cfg_gmv_tb_a *tb_a; + struct udma_cfg_gmv_tb_b *tb_b; + struct udma_cmq_desc desc[2]; + uint16_t smac_l; + + tb_a = (struct udma_cfg_gmv_tb_a *)desc[0].data; + tb_b = (struct udma_cfg_gmv_tb_b *)desc[1].data; + + udma_cmq_setup_basic_desc(&desc[0], UDMA_OPC_CFG_GMV_TBL, false); + desc[0].flag |= cpu_to_le16(UDMA_CMD_FLAG_NEXT); + udma_cmq_setup_basic_desc(&desc[1], UDMA_OPC_CFG_GMV_TBL, false); + + smac_l = + *(uint16_t *)&udma_dev->uboe.netdevs[0]->dev_addr[SMAC_L_SHIFT]; + udma_set_field(tb_a->vf_type_vlan_smac, CFG_GMV_TB_VF_SMAC_L_M, + CFG_GMV_TB_VF_SMAC_L_S, smac_l); + tb_a->vf_smac_h = + *(uint32_t *)&udma_dev->uboe.netdevs[0]->dev_addr[SMAC_H_SHIFT]; + udma_set_field(tb_a->vf_type_vlan_smac, CFG_GMV_TB_VF_SGID_TYPE_M, + CFG_GMV_TB_VF_SGID_TYPE_S, sgid_type); + memcpy(tb_a, &eid, sizeof(eid)); + udma_set_bit(tb_a->vf_type_vlan_smac, CFG_GMV_TB_VF_PATTERN_S, 0); + tb_b->vf_id = 0; + + return udma_cmq_send(udma_dev, desc, CFG_GMV_TBL_CMD_NUM); +} + +static void udma_fill_eid_addr(struct udma_eid_tbl_entry_cmd *eid_entry, + union ubcore_eid eid) +{ + int i; + + /* big endian */ + for (i = 0; i < UDMA_EID_SIZE_IDX; i++) + eid_entry->eid_addr[i] = (*(((uint32_t *)eid.raw) + i)); +} + +static int set_eid_table(struct udma_dev *udma_dev, union ubcore_eid eid) +{ + struct udma_eid_tbl_entry_cmd *eid_entry; + struct udma_cmq_desc desc = {}; + uint8_t resp_code; + int ret; + + eid_entry = (struct udma_eid_tbl_entry_cmd *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_DEID_TBL_ADD, false); + udma_set_field(eid_entry->eid_ad, UDMA_EID_TB_VFID_M, + UDMA_EID_TB_VFID_S, 0); + udma_fill_eid_addr(eid_entry, eid); + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) { + dev_err(udma_dev->dev, "Send set eid table cmd failed.\n"); + return ret; + } + + resp_code = (le32_to_cpu(desc.data[0])) & 0xff; + if (resp_code == UDMA_EID_TB_RES_SUCCESS || + resp_code == UDMA_EID_TB_RES_MODIFY) + return 0; + else + return -EIO; +} + +static int udma_hw_set_eid(struct udma_dev *udma_dev, union ubcore_eid eid) +{ + int ret; + + ret = config_gmv_table(udma_dev, eid); + if (ret) { + dev_err(udma_dev->dev, "Set EID to GMV table failed.\n"); + return ret; + } + + ret = set_eid_table(udma_dev, eid); + if (ret) + dev_err(udma_dev->dev, "Set EID table failed.\n"); + + return ret; +} + +static void func_clr_hw_resetting_state(struct udma_dev *udma_dev, + struct hnae3_handle *handle) +{ + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + uint64_t end; + + udma_dev->dis_db = true; + + dev_warn(udma_dev->dev, + "Func clear is pending, device in resetting state.\n"); + end = UDMA_HW_RST_TIMEOUT; + while (end) { + if (!ops->get_hw_reset_stat(handle)) { + udma_dev->is_reset = true; + dev_info(udma_dev->dev, + "Func clear success after reset.\n"); + return; + } + msleep(UDMA_HW_RST_COMPLETION_WAIT); + end -= UDMA_HW_RST_COMPLETION_WAIT; + } + + dev_warn(udma_dev->dev, "Func clear failed.\n"); +} + +static void func_clr_sw_resetting_state(struct udma_dev *udma_dev, + struct hnae3_handle *handle) +{ + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + uint64_t end; + + udma_dev->dis_db = true; + + dev_warn(udma_dev->dev, + "Func clear is pending, device in resetting state.\n"); + end = UDMA_HW_RST_TIMEOUT; + while (end) { + if (ops->ae_dev_reset_cnt(handle) != + udma_dev->reset_cnt) { + udma_dev->is_reset = true; + dev_info(udma_dev->dev, + "Func clear success after sw reset\n"); + return; + } + msleep(UDMA_HW_RST_COMPLETION_WAIT); + end -= UDMA_HW_RST_COMPLETION_WAIT; + } + + dev_warn(udma_dev->dev, + "Func clear failed because of unfinished sw reset\n"); +} + +static void udma_func_clr_rst_proc(struct udma_dev *udma_dev, int retval, + int flag) +{ + struct udma_priv *priv = (struct udma_priv *)udma_dev->priv; + struct hnae3_handle *handle = priv->handle; + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + + if (ops->ae_dev_reset_cnt(handle) != udma_dev->reset_cnt) { + udma_dev->dis_db = true; + udma_dev->is_reset = true; + dev_info(udma_dev->dev, "Func clear success after reset.\n"); + return; + } + + if (ops->get_hw_reset_stat(handle)) { + func_clr_hw_resetting_state(udma_dev, handle); + return; + } + + if (ops->ae_dev_resetting(handle) && + handle->udmainfo.instance_state == UDMA_STATE_INIT) { + func_clr_sw_resetting_state(udma_dev, handle); + return; + } + + if (retval && !flag) + dev_warn(udma_dev->dev, + "Func clear read failed, ret = %d.\n", retval); + + dev_warn(udma_dev->dev, "Func clear failed.\n"); +} + +static bool check_device_is_in_reset(struct udma_dev *udma_dev) +{ + struct udma_priv *priv = (struct udma_priv *)udma_dev->priv; + struct hnae3_handle *handle = priv->handle; + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + + if (udma_dev->reset_cnt != ops->ae_dev_reset_cnt(handle)) + return true; + + if (ops->get_hw_reset_stat(handle)) + return true; + + if (ops->ae_dev_resetting(handle)) + return true; + + return false; +} + +static void __udma_function_clear(struct udma_dev *udma_dev, int vf_id) +{ + bool fclr_write_fail_flag = false; + struct udma_func_clear *resp; + struct udma_cmq_desc desc; + uint64_t end; + int ret = 0; + + if (check_device_is_in_reset(udma_dev)) + goto out; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_FUNC_CLEAR, false); + resp = (struct udma_func_clear *)desc.data; + resp->rst_funcid_en = cpu_to_le32(vf_id); + + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) { + fclr_write_fail_flag = true; + dev_err(udma_dev->dev, "Func clear write failed, ret = %d.\n", + ret); + goto out; + } + + msleep(UDMA_READ_FUNC_CLEAR_FLAG_INTERVAL); + end = UDMA_FUNC_CLEAR_TIMEOUT_MSECS; + while (end) { + if (check_device_is_in_reset(udma_dev)) + goto out; + msleep(UDMA_READ_FUNC_CLEAR_FLAG_FAIL_WAIT); + end -= UDMA_READ_FUNC_CLEAR_FLAG_FAIL_WAIT; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_FUNC_CLEAR, + true); + + resp->rst_funcid_en = cpu_to_le32(vf_id); + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) + continue; + + if (udma_get_bit(resp->func_done, FUNC_CLEAR_RST_FUN_DONE_S)) { + if (vf_id == 0) + udma_dev->is_reset = true; + return; + } + } + +out: + udma_func_clr_rst_proc(udma_dev, ret, fclr_write_fail_flag); +} + +static void udma_free_vf_resource(struct udma_dev *udma_dev, int vf_id) +{ + enum udma_opcode_type opcode = UDMA_OPC_ALLOC_VF_RES; + struct udma_cmq_desc desc[2]; + struct udma_cmq_req *req_a; + + req_a = (struct udma_cmq_req *)desc[0].data; + udma_cmq_setup_basic_desc(&desc[0], opcode, false); + desc[0].flag |= cpu_to_le16(UDMA_CMD_FLAG_NEXT); + udma_cmq_setup_basic_desc(&desc[1], opcode, false); + udma_reg_write(req_a, FUNC_RES_A_VF_ID, vf_id); + udma_cmq_send(udma_dev, desc, 2); +} + +static void udma_function_clear(struct udma_dev *udma_dev) +{ + int i; + + if (udma_dev->cmd.state == UDMA_CMDQ_STATE_FATAL_ERR) + return; + + for (i = udma_dev->func_num - 1; i >= 0; i--) { + __udma_function_clear(udma_dev, i); + if (i != 0) + udma_free_vf_resource(udma_dev, i); + } +} + +static void config_llm_table(struct udma_buf *data_buf, void *cfg_buf) +{ + uint64_t *entry = (uint64_t *)cfg_buf; + uint32_t i, next_ptr, page_num; + dma_addr_t addr; + uint64_t val; + + page_num = data_buf->npages; + for (i = 0; i < page_num; i++) { + addr = udma_buf_page(data_buf, i); + if (i == (page_num - 1)) + next_ptr = 0; + else + next_ptr = i + 1; + + val = UDMA_EXT_LLM_ENTRY(addr, (uint64_t)next_ptr); + entry[i] = cpu_to_le64(val); + } +} + +static int set_llm_cfg_to_hw(struct udma_dev *udma_dev, + struct udma_link_table *table) +{ + struct udma_cmq_desc desc[2]; + struct udma_cmq_req *r_a = (struct udma_cmq_req *)desc[0].data; + struct udma_cmq_req *r_b = (struct udma_cmq_req *)desc[1].data; + struct udma_buf *buf = table->buf; + enum udma_opcode_type opcode; + dma_addr_t addr; + + opcode = UDMA_OPC_CFG_EXT_LLM; + udma_cmq_setup_basic_desc(&desc[0], opcode, false); + desc[0].flag |= cpu_to_le16(UDMA_CMD_FLAG_NEXT); + udma_cmq_setup_basic_desc(&desc[1], opcode, false); + + udma_reg_write(r_a, CFG_LLM_A_BA_L, lower_32_bits(table->table.map)); + udma_reg_write(r_a, CFG_LLM_A_BA_H, upper_32_bits(table->table.map)); + udma_reg_write(r_a, CFG_LLM_A_DEPTH, buf->npages); + udma_reg_write(r_a, CFG_LLM_A_PG_SZ, + to_hr_hw_page_shift(buf->page_shift)); + udma_reg_enable(r_a, CFG_LLM_A_INIT_EN); + + addr = to_hr_hw_page_addr(udma_buf_page(buf, 0)); + udma_reg_write(r_a, CFG_LLM_A_HEAD_BA_L, lower_32_bits(addr)); + udma_reg_write(r_a, CFG_LLM_A_HEAD_BA_H, upper_32_bits(addr)); + udma_reg_write(r_a, CFG_LLM_A_HEAD_NXT_PTR, 1); + udma_reg_write(r_a, CFG_LLM_A_HEAD_PTR, 0); + + addr = to_hr_hw_page_addr(udma_buf_page(buf, buf->npages - 1)); + udma_reg_write(r_b, CFG_LLM_B_TAIL_BA_L, lower_32_bits(addr)); + udma_reg_write(r_b, CFG_LLM_B_TAIL_BA_H, upper_32_bits(addr)); + udma_reg_write(r_b, CFG_LLM_B_TAIL_PTR, buf->npages - 1); + + return udma_cmq_send(udma_dev, desc, 2); +} + +static struct udma_link_table * +alloc_link_table_buf(struct udma_dev *udma_dev) +{ + struct udma_priv *priv = (struct udma_priv *)udma_dev->priv; + uint32_t pg_shift, size, min_size; + struct udma_link_table *link_tbl; + + link_tbl = &priv->ext_llm; + pg_shift = udma_dev->caps.llm_buf_pg_sz + PAGE_SHIFT; + size = udma_dev->caps.num_qps * UDMA_EXT_LLM_ENTRY_SZ; + min_size = UDMA_EXT_LLM_MIN_PAGES(udma_dev->caps.sl_num) << pg_shift; + + /* Alloc data table */ + size = max(size, min_size); + link_tbl->buf = udma_buf_alloc(udma_dev, size, pg_shift, 0); + if (IS_ERR(link_tbl->buf)) + return ERR_PTR(-ENOMEM); + + /* Alloc config table */ + size = link_tbl->buf->npages * sizeof(uint64_t); + link_tbl->table.buf = dma_alloc_coherent(udma_dev->dev, size, + &link_tbl->table.map, + GFP_KERNEL); + if (!link_tbl->table.buf) { + udma_buf_free(udma_dev, link_tbl->buf); + return ERR_PTR(-ENOMEM); + } + + return link_tbl; +} + +static void free_link_table_buf(struct udma_dev *udma_dev, + struct udma_link_table *tbl) +{ + if (tbl->buf) { + uint32_t size = tbl->buf->npages * sizeof(uint64_t); + + dma_free_coherent(udma_dev->dev, size, tbl->table.buf, + tbl->table.map); + } + + udma_buf_free(udma_dev, tbl->buf); +} + +static int udma_init_link_table(struct udma_dev *udma_dev) +{ + struct udma_link_table *link_tbl; + int ret; + + link_tbl = alloc_link_table_buf(udma_dev); + if (IS_ERR(link_tbl)) + return -ENOMEM; + + if (WARN_ON(link_tbl->buf->npages > UDMA_EXT_LLM_MAX_DEPTH)) { + ret = -EINVAL; + goto err_alloc; + } + + config_llm_table(link_tbl->buf, link_tbl->table.buf); + ret = set_llm_cfg_to_hw(udma_dev, link_tbl); + if (ret) + goto err_alloc; + + return 0; + +err_alloc: + free_link_table_buf(udma_dev, link_tbl); + return ret; +} + +static void udma_free_link_table(struct udma_dev *udma_dev) +{ + struct udma_priv *priv = (struct udma_priv *)udma_dev->priv; + + free_link_table_buf(udma_dev, &priv->ext_llm); +} + +static void free_dip_list(struct udma_dev *udma_dev) +{ + struct udma_dip *u_dip; + struct udma_dip *tmp; + unsigned long flags; + + spin_lock_irqsave(&udma_dev->dip_list_lock, flags); + + list_for_each_entry_safe(u_dip, tmp, &udma_dev->dip_list, node) { + list_del(&u_dip->node); + kfree(u_dip); + } + + spin_unlock_irqrestore(&udma_dev->dip_list_lock, flags); +} + +static int udma_get_reset_page(struct udma_dev *dev) +{ + dev->reset_page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!dev->reset_page) + return -ENOMEM; + + dev->reset_kaddr = vmap(&dev->reset_page, 1, VM_MAP, PAGE_KERNEL); + if (!dev->reset_kaddr) + goto err_vmap; + + return 0; + +err_vmap: + put_page(dev->reset_page); + return -ENOMEM; +} + +static void udma_put_reset_page(struct udma_dev *dev) +{ + vunmap(dev->reset_kaddr); + dev->reset_kaddr = NULL; + put_page(dev->reset_page); + dev->reset_page = NULL; +} + +static int udma_clear_extdb_list_info(struct udma_dev *udma_dev) +{ + struct udma_cmq_desc desc; + int ret; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_CLEAR_EXTDB_LIST_INFO, + false); + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) + dev_err(udma_dev->dev, + "failed to clear extended doorbell info, ret = %d.\n", + ret); + + return ret; +} + +int get_hem_table(struct udma_dev *udma_dev) +{ + uint32_t qpc_count; + uint32_t cqc_count; + uint32_t gmv_count; + uint32_t i; + int ret; + + /* Alloc memory for source address table buffer space chunk */ + for (gmv_count = 0; gmv_count < udma_dev->caps.gmv_entry_num; + gmv_count++) { + ret = udma_table_get(udma_dev, &udma_dev->gmv_table, gmv_count); + if (ret) + goto err_gmv_failed; + } + + /* Alloc memory for QPC Timer buffer space chunk */ + for (qpc_count = 0; qpc_count < udma_dev->caps.qpc_timer_bt_num; + qpc_count++) { + ret = udma_table_get(udma_dev, &udma_dev->qpc_timer_table, + qpc_count); + if (ret) { + dev_err(udma_dev->dev, "QPC Timer get failed\n"); + goto err_qpc_timer_failed; + } + } + + /* Alloc memory for CQC Timer buffer space chunk */ + for (cqc_count = 0; cqc_count < udma_dev->caps.cqc_timer_bt_num; + cqc_count++) { + ret = udma_table_get(udma_dev, &udma_dev->cqc_timer_table, + cqc_count); + if (ret) { + dev_err(udma_dev->dev, "CQC Timer get failed\n"); + goto err_cqc_timer_failed; + } + } + + return 0; + +err_cqc_timer_failed: + for (i = 0; i < cqc_count; i++) + udma_table_put(udma_dev, &udma_dev->cqc_timer_table, i); + +err_qpc_timer_failed: + for (i = 0; i < qpc_count; i++) + udma_table_put(udma_dev, &udma_dev->qpc_timer_table, i); + +err_gmv_failed: + for (i = 0; i < gmv_count; i++) + udma_table_put(udma_dev, &udma_dev->gmv_table, i); + + return ret; +} + +void put_hem_table(struct udma_dev *udma_dev) +{ + uint32_t i; + + for (i = 0; i < udma_dev->caps.gmv_entry_num; i++) + udma_table_put(udma_dev, &udma_dev->gmv_table, i); + + + for (i = 0; i < udma_dev->caps.qpc_timer_bt_num; i++) + udma_table_put(udma_dev, &udma_dev->qpc_timer_table, i); + + for (i = 0; i < udma_dev->caps.cqc_timer_bt_num; i++) + udma_table_put(udma_dev, &udma_dev->cqc_timer_table, i); +} + +static int udma_hw_init(struct udma_dev *udma_dev) +{ + int ret; + + ret = udma_get_reset_page(udma_dev); + if (ret) { + dev_err(udma_dev->dev, + "get reset page failed, ret = %d.\n", ret); + return ret; + } + + /* UDMA requires the extdb info to be cleared before using */ + ret = udma_clear_extdb_list_info(udma_dev); + if (ret) + goto err_clear_extdb_failed; + + ret = get_hem_table(udma_dev); + if (ret) + goto err_clear_extdb_failed; + + ret = udma_init_link_table(udma_dev); + if (ret) { + dev_err(udma_dev->dev, "failed to init llm, ret = %d.\n", ret); + goto err_llm_init_failed; + } + + return 0; + +err_llm_init_failed: + put_hem_table(udma_dev); +err_clear_extdb_failed: + udma_put_reset_page(udma_dev); + + return ret; +} + +static void udma_hw_exit(struct udma_dev *udma_dev) +{ + udma_function_clear(udma_dev); + + udma_free_link_table(udma_dev); + + put_hem_table(udma_dev); + free_dip_list(udma_dev); +} + +static int get_op_for_set_hem(uint32_t type, int step_idx, uint16_t *mbox_op, + bool is_create) +{ + uint16_t op; + + switch (type) { + case HEM_TYPE_QPC: + op = is_create ? UDMA_CMD_WRITE_QPC_BT0 : + UDMA_CMD_DESTROY_QPC_BT0; + break; + case HEM_TYPE_MTPT: + op = is_create ? UDMA_CMD_WRITE_MPT_BT0 : + UDMA_CMD_DESTROY_MPT_BT0; + break; + case HEM_TYPE_CQC: + op = is_create ? UDMA_CMD_WRITE_CQC_BT0 : + UDMA_CMD_DESTROY_CQC_BT0; + break; + case HEM_TYPE_SRQC: + op = is_create ? UDMA_CMD_WRITE_SRQC_BT0 : + UDMA_CMD_DESTROY_SRQC_BT0; + break; + case HEM_TYPE_SCCC: + op = is_create ? UDMA_CMD_WRITE_SCCC_BT0 : UDMA_CMD_RESERVED; + break; + case HEM_TYPE_QPC_TIMER: + op = is_create ? UDMA_CMD_WRITE_QPC_TIMER_BT0 : + UDMA_CMD_RESERVED; + break; + case HEM_TYPE_CQC_TIMER: + op = is_create ? UDMA_CMD_WRITE_CQC_TIMER_BT0 : + UDMA_CMD_RESERVED; + break; + case HEM_TYPE_GMV: + op = is_create ? UDMA_CMD_RESERVED : UDMA_CMD_RESERVED; + break; + default: + return -EINVAL; + } + + if (op != UDMA_CMD_RESERVED) + *mbox_op = op + step_idx; + + return 0; +} + +static int config_gmv_ba_to_hw(struct udma_dev *udma_dev, uint64_t obj, + dma_addr_t base_addr) +{ + struct udma_cmq_desc desc; + struct udma_cmq_req *req = (struct udma_cmq_req *)desc.data; + uint32_t idx = obj / (UDMA_PAGE_SIZE / udma_dev->caps.gmv_entry_sz); + uint64_t addr = to_hr_hw_page_addr(base_addr); + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_CFG_GMV_BT, false); + + udma_reg_write(req, CFG_GMV_BT_BA_L, lower_32_bits(addr)); + udma_reg_write(req, CFG_GMV_BT_BA_H, upper_32_bits(addr)); + udma_reg_write(req, CFG_GMV_BT_IDX, idx); + udma_reg_write(req, CFG_GMV_BT_VF_ID, 0); + + return udma_cmq_send(udma_dev, &desc, 1); +} + +static int config_hem_ba_to_hw(struct udma_dev *udma_dev, uint64_t obj, + uint64_t base_addr, uint16_t op) +{ + struct udma_cmd_mailbox *mbox = udma_alloc_cmd_mailbox(udma_dev); + struct udma_cmq_desc desc; + struct udma_mbox *mb; + int ret; + + if (IS_ERR_OR_NULL(mbox)) + return -ENOMEM; + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mbox_desc_init(mb, base_addr, mbox->dma, obj, op); + ret = udma_cmd_mbox(udma_dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); + if (ret) + dev_err(udma_dev->dev, "[mailbox cmd] config hem ba failed.\n"); + + udma_free_cmd_mailbox(udma_dev, mbox); + + return ret; +} + +static int set_hem_to_hw(struct udma_dev *udma_dev, int obj, + dma_addr_t base_addr, + uint32_t hem_type, int step_idx) +{ + bool is_create = true; + uint16_t op; + int ret; + + if (unlikely(hem_type == HEM_TYPE_GMV)) + return config_gmv_ba_to_hw(udma_dev, obj, base_addr); + + if (unlikely(hem_type == HEM_TYPE_SCCC && step_idx)) + return 0; + + ret = get_op_for_set_hem(hem_type, step_idx, &op, is_create); + if (ret < 0) + return ret; + + return config_hem_ba_to_hw(udma_dev, obj, base_addr, op); +} + +static int udma_set_hem(struct udma_dev *udma_dev, struct udma_hem_table *table, + int obj, int step_idx) +{ + struct udma_hem_iter iter; + struct udma_hem_mhop mhop; + uint64_t mhop_obj = obj; + uint32_t chunk_ba_num; + struct udma_hem *hem; + uint64_t hem_idx = 0; + uint64_t l1_idx = 0; + uint64_t bt_ba = 0; + uint32_t hop_num; + int i, j, k; + int ret; + + if (!udma_check_whether_mhop(udma_dev, table->type)) + return 0; + + udma_calc_hem_mhop(udma_dev, table, &mhop_obj, &mhop); + i = mhop.l0_idx; + j = mhop.l1_idx; + k = mhop.l2_idx; + hop_num = mhop.hop_num; + chunk_ba_num = mhop.bt_chunk_size / BA_BYTE_LEN; + + if (hop_num == 2) { + hem_idx = i * chunk_ba_num * chunk_ba_num + j * chunk_ba_num + + k; + l1_idx = i * chunk_ba_num + j; + } else if (hop_num == 1) { + hem_idx = i * chunk_ba_num + j; + } else if (hop_num == UDMA_HOP_NUM_0) { + hem_idx = i; + } + + if (table->type == HEM_TYPE_SCCC) + obj = mhop.l0_idx; + + if (check_whether_last_step(hop_num, step_idx)) { + hem = table->hem[hem_idx]; + for (udma_hem_first(hem, &iter); + !udma_hem_last(&iter); udma_hem_next(&iter)) { + bt_ba = udma_hem_addr(&iter); + ret = set_hem_to_hw(udma_dev, obj, bt_ba, table->type, + step_idx); + } + } else { + if (step_idx == 0) + bt_ba = table->bt_l0_dma_addr[i]; + else if (step_idx == 1 && hop_num == 2) + bt_ba = table->bt_l1_dma_addr[l1_idx]; + + ret = set_hem_to_hw(udma_dev, obj, bt_ba, table->type, + step_idx); + } + + return ret; +} + +static int udma_clear_hem(struct udma_dev *udma_dev, + struct udma_hem_table *table, + int obj, int step_idx) +{ + uint16_t op = UDMA_CMD_RESERVED; + bool is_create = false; + int ret; + + if (!udma_check_whether_mhop(udma_dev, table->type)) + return 0; + + ret = get_op_for_set_hem(table->type, step_idx, &op, is_create); + if (ret < 0 || op == UDMA_CMD_RESERVED) + return ret; + + return config_hem_ba_to_hw(udma_dev, obj, 0, op); +} + +static const struct udma_hw udma_hw = { + .cmq_init = udma_cmq_init, + .cmq_exit = udma_cmq_exit, + .hw_profile = udma_profile, + .hw_init = udma_hw_init, + .hw_exit = udma_hw_exit, + .post_mbox = udma_post_mbox, + .poll_mbox_done = udma_poll_mbox_done, + .chk_mbox_avail = udma_chk_mbox_is_avail, + .set_hem = udma_set_hem, + .clear_hem = udma_clear_hem, + .set_eid = udma_hw_set_eid, + .init_eq = udma_init_eq_table, + .cleanup_eq = udma_cleanup_eq_table, +}; + +bool udma_is_virtual_func(struct pci_dev *pdev) +{ + uint32_t dev_id = pdev->device; + + switch (dev_id) { + case HNAE3_DEV_ID_UDMA_OVER_UBL: + case HNAE3_DEV_ID_UDMA: + return false; + case HNAE3_DEV_ID_UDMA_OVER_UBL_VF: + return true; + default: + dev_warn(&pdev->dev, "un-recognized pci deviced-id %x.\n", + dev_id); + } + + return false; +} + +static void udma_get_cfg(struct udma_dev *udma_dev, + struct hnae3_handle *handle) +{ + struct udma_priv *priv = (struct udma_priv *)udma_dev->priv; + int i; + + udma_dev->pci_dev = handle->pdev; + udma_dev->dev = &handle->pdev->dev; + udma_dev->hw = &udma_hw; + + /* Get info from NIC driver. */ + udma_dev->reg_base = handle->udmainfo.udma_io_base; + udma_dev->caps.num_ports = 1; + udma_dev->uboe.netdevs[0] = handle->udmainfo.netdev; + + for (i = 0; i < handle->udmainfo.num_vectors; i++) + udma_dev->irq[i] = pci_irq_vector(handle->pdev, + i + + handle->udmainfo.base_vector); + + /* cmd issue mode: 0 is poll, 1 is event */ + udma_dev->cmd_mod = 0; + + udma_dev->reset_cnt = handle->ae_algo->ops->ae_dev_reset_cnt(handle); + priv->handle = handle; +} + +static int __udma_init_instance(struct hnae3_handle *handle) +{ + struct udma_dev *udma_dev; + int ret; + + udma_dev = kzalloc(sizeof(*udma_dev), GFP_KERNEL); + if (ZERO_OR_NULL_PTR(udma_dev)) + return -ENOMEM; + + udma_dev->priv = kzalloc(sizeof(struct udma_priv), GFP_KERNEL); + if (ZERO_OR_NULL_PTR(udma_dev->priv)) { + ret = -ENOMEM; + goto error_failed_kzalloc; + } + + udma_get_cfg(udma_dev, handle); + + ret = udma_hnae_client_init(udma_dev); + if (ret) { + dev_err(udma_dev->dev, "UDMA Engine init failed(%d)!\n", ret); + goto error_failed_get_cfg; + } + handle->priv = udma_dev; + + if (dfx_switch) { + ret = udma_dfx_init(udma_dev); + if (ret) { + dev_err(udma_dev->dev, "UDMA dfx init failed(%d)!\n", + ret); + goto error_failed_dfx_init; + } + } + + ret = udma_register_cc_sysfs(udma_dev); + if (ret) { + dev_err(udma_dev->dev, + "UDMA congest control init failed(%d)!\n", ret); + goto error_failed_cc_sysfs; + } + + return 0; +error_failed_cc_sysfs: + if (dfx_switch) + udma_dfx_uninit(udma_dev); +error_failed_dfx_init: + udma_hnae_client_exit(udma_dev); +error_failed_get_cfg: + kfree(udma_dev->priv); +error_failed_kzalloc: + kfree(udma_dev); + + return ret; +} + +static void __udma_uninit_instance(struct hnae3_handle *handle, + bool reset) +{ + struct udma_dev *udma_dev = handle->priv; + + if (!udma_dev) + return; + + udma_unregister_cc_sysfs(udma_dev); + + if (dfx_switch) + udma_dfx_uninit(handle->priv); + + handle->priv = NULL; + + udma_hnae_client_exit(udma_dev); + kfree(udma_dev->priv); + kfree(udma_dev); +} + +static int udma_init_instance(struct hnae3_handle *handle) +{ + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + struct device *dev = &handle->pdev->dev; + const struct pci_device_id *id; + int ret; + + handle->udmainfo.instance_state = UDMA_STATE_INIT; + + id = pci_match_id(udma_hw_pci_tbl, handle->pdev); + if (!id) + return 0; + + ret = __udma_init_instance(handle); + if (ret) { + handle->udmainfo.instance_state = UDMA_STATE_NON_INIT; + if (ret == -EOPNOTSUPP) + return ret; + + dev_err(dev, "UDMA instance init failed! ret = %d\n", ret); + if (ops->ae_dev_resetting(handle) || + ops->get_hw_reset_stat(handle)) + goto reset_chk_err; + else + return ret; + } + + handle->udmainfo.instance_state = UDMA_STATE_INITED; + + return 0; + +reset_chk_err: + dev_err(dev, "Device is being reset, please retry later.\n"); + + return -EBUSY; +} + +static void udma_uninit_instance(struct hnae3_handle *handle, bool reset) +{ + if (handle->udmainfo.instance_state != UDMA_STATE_INITED) + return; + + handle->udmainfo.instance_state = UDMA_STATE_UNINIT; + + __udma_uninit_instance(handle, reset); + + handle->udmainfo.instance_state = UDMA_STATE_NON_INIT; +} + +static void udma_reset_notify_user(struct udma_dev *dev) +{ + struct udma_reset_state *state; + + state = (struct udma_reset_state *)dev->reset_kaddr; + + state->reset_state = UDMA_IS_RESETTING; + /* Ensure reset state was flushed in memory */ + wmb(); +} + +static int udma_reset_notify_down(struct hnae3_handle *handle) +{ + struct udma_dev *dev; + + if (handle->udmainfo.instance_state != UDMA_STATE_INITED) { + set_bit(UDMA_RST_DIRECT_RETURN, &handle->udmainfo.state); + return 0; + } + + handle->udmainfo.reset_state = UDMA_STATE_RST_DOWN; + clear_bit(UDMA_RST_DIRECT_RETURN, &handle->udmainfo.state); + + dev = handle->priv; + if (!dev) + return 0; + + dev->dis_db = true; + + udma_reset_notify_user(dev); + + return 0; +} + +static int udma_reset_notify_init(struct hnae3_handle *handle) +{ + struct device *dev = &handle->pdev->dev; + int ret; + + if (test_and_clear_bit(UDMA_RST_DIRECT_RETURN, + &handle->udmainfo.state)) { + handle->udmainfo.reset_state = UDMA_STATE_RST_INITED; + return 0; + } + + handle->udmainfo.reset_state = UDMA_STATE_RST_INIT; + + dev_info(&handle->pdev->dev, "In reset process UDMA client reinit.\n"); + ret = __udma_init_instance(handle); + if (ret) { + /* when reset notify type is HNAE3_INIT_CLIENT In reset notify + * callback function, UB Engine reinitialize. If UDMA reinit + * failed, we should inform NIC driver. + */ + handle->priv = NULL; + dev_err(dev, "In reset process UDMA reinit failed %d.\n", ret); + } else { + handle->udmainfo.reset_state = UDMA_STATE_RST_INITED; + dev_info(dev, "Reset done, UDMA client reinit finished.\n"); + } + + return ret; +} + +static int udma_reset_notify_uninit(struct hnae3_handle *handle) +{ + if (test_bit(UDMA_RST_DIRECT_RETURN, &handle->udmainfo.state)) + return 0; + + handle->udmainfo.reset_state = UDMA_STATE_RST_UNINIT; + dev_info(&handle->pdev->dev, "In reset process UDMA client uninit.\n"); + msleep(UDMA_HW_RST_UNINT_DELAY); + __udma_uninit_instance(handle, false); + + return 0; +} + +static int udma_reset_notify(struct hnae3_handle *handle, + enum hnae3_reset_notify_type type) +{ + int ret = 0; + + switch (type) { + case HNAE3_DOWN_CLIENT: + ret = udma_reset_notify_down(handle); + break; + case HNAE3_INIT_CLIENT: + ret = udma_reset_notify_init(handle); + break; + case HNAE3_UNINIT_CLIENT: + ret = udma_reset_notify_uninit(handle); + break; + default: + break; + } + + return ret; +} + +static void udma_link_status_change(struct hnae3_handle *handle, bool linkup) +{ + struct net_device *net_dev; + struct ubcore_event event; + struct udma_dev *dev; + uint32_t port_id; + + dev = handle->priv; + + if (IS_ERR_OR_NULL(dev)) { + pr_err("[hns3-udma:link_status_change]: Invalid dev!\n"); + return; + } + + for (port_id = 0; port_id < dev->caps.num_ports; port_id++) { + net_dev = dev->uboe.netdevs[port_id]; + if (!net_dev) { + dev_err(dev->dev, "Find netdev %u failed!\n", port_id); + return; + } + + if (net_dev == handle->udmainfo.netdev) + break; + } + + if (port_id == dev->caps.num_ports) { + dev_err(dev->dev, "Cannot find netdev!\n"); + return; + } + + if (linkup) + event.event_type = UBCORE_EVENT_PORT_ACTIVE; + else + event.event_type = UBCORE_EVENT_PORT_ERR; + + event.ub_dev = &dev->ub_dev; + event.element.port_id = port_id; + ubcore_dispatch_async_event(&event); +} + +static const struct hnae3_client_ops udma_ops = { + .init_instance = udma_init_instance, + .uninit_instance = udma_uninit_instance, + .link_status_change = udma_link_status_change, + .reset_notify = udma_reset_notify, +}; + +static struct hnae3_client udma_client = { + .name = "udma", + .type = HNAE3_CLIENT_UDMA, + .ops = &udma_ops, +}; + +static int __init udma_init(void) +{ + return hnae3_register_client(&udma_client); +} + +static void __exit udma_exit(void) +{ + hnae3_unregister_client(&udma_client); +} + +module_init(udma_init); +module_exit(udma_exit); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_DESCRIPTION("UBUS UDMA Driver"); + +module_param(dfx_switch, bool, 0444); +MODULE_PARM_DESC(dfx_switch, + "Set whether to enable the udma_dfx, default: 0(0:off, 1:on)"); diff --git a/drivers/ub/hw/hns3/hns3_udma_hw.h b/drivers/ub/hw/hns3/hns3_udma_hw.h new file mode 100644 index 0000000000000000000000000000000000000000..17e1c2750cd847b4942ffcdec7ab0046f6286c07 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_hw.h @@ -0,0 +1,539 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_HW_H +#define _UDMA_HW_H + +#include "urma/ubcore_types.h" + +extern bool dfx_switch; + +struct udma_query_version { + uint16_t udma_vendor_id; + uint16_t udma_hw_version; + uint32_t rsv[5]; +}; + +struct udma_query_fw_info { + uint32_t fw_ver; + uint32_t rsv[5]; +}; + +struct udma_func_clear { + uint32_t rst_funcid_en; + uint32_t func_done; + uint32_t rsv[4]; +}; + +struct udma_pf_func_info { + uint32_t own_func_num; + uint32_t own_mac_id; + uint32_t rsv[4]; +}; + +struct udma_cmq_req { + uint32_t data[6]; +}; + +struct udma_vf_switch { + uint32_t udma_sel; + uint32_t fun_id; + uint32_t cfg; + uint32_t resv1; + uint32_t resv2; + uint32_t resv3; +}; + +struct udma_hw_id_query_cmq { + uint8_t chip_id; + uint8_t die_id; + uint8_t mac_id; + uint8_t reserved; + uint32_t func_id; + uint32_t rsv[4]; +}; + +struct udma_query_oor_cmq { + uint8_t oor_en; + uint8_t reorder_cq_buffer_en; + uint8_t reorder_cap; + uint8_t reorder_cq_shift; + uint32_t on_flight_size; + uint8_t dynamic_ack_timeout; + uint8_t rsv[15]; +}; + +#define UDMA_MAX_MTT_SEGS 0x1000000 +#define UDMA_MAX_SRQWQE_SEGS 0x1000000 +#define UDMA_MAX_IDX_SEGS 0x1000000 +#define UDMA_MTT_ENTRY_SZ 64 +#define UDMA_EID_SIZE_IDX 4 + +#define UDMA_PBL_HOP_NUM 2 +#define UDMA_EQE_HOP_NUM 2 + +#define UDMA_QPC_TIMER_ENTRY_SZ PAGE_SIZE +#define UDMA_CQC_TIMER_ENTRY_SZ PAGE_SIZE + +#define UDMA_EQE_SIZE 0x40 +#define UDMA_EQ_OVER_IGNORE_0 0 +#define UDMA_EQ_COALESCE_0 0 +#define UDMA_EQ_ALWAYS_ARMED 3 +#define UDMA_EQN_M GENMASK(23, 0) + +#define UDMA_CEQ_CEQE_OWNER_S 31 +#define UDMA_AEQ_AEQE_OWNER_S 31 + +#define NIC_ICL_SWITCH_CMD_UDMA_SEL_SHIFT 0 +#define NIC_ICL_SWITCH_CMD_UDMA_SEL BIT(NIC_ICL_SWITCH_CMD_UDMA_SEL_SHIFT) + +#define VF_SWITCH_DATA_FUN_ID_VF_ID_S 3 +#define VF_SWITCH_DATA_FUN_ID_VF_ID_M GENMASK(10, 3) + +#define VF_SWITCH_DATA_CFG_ALW_LPBK_S 1 +#define VF_SWITCH_DATA_CFG_ALW_LCL_LPBK_S 2 +#define VF_SWITCH_DATA_CFG_ALW_DST_OVRD_S 3 + +#define CMQ_REQ_FIELD_LOC(h, l) ((uint64_t)(h) << 32 | (l)) + +/* Fields of UDMA_OPC_CFG_GLOBAL_PARAM */ +#define CFG_GLOBAL_PARAM_1US_CYCLES CMQ_REQ_FIELD_LOC(9, 0) +#define CFG_GLOBAL_PARAM_UDP_PORT CMQ_REQ_FIELD_LOC(31, 16) + +#define FUNC_CLEAR_RST_FUN_DONE_S 0 +#define UDMA_FUNC_CLEAR_TIMEOUT_MSECS (249 * 2 * 100) +#define UDMA_READ_FUNC_CLEAR_FLAG_INTERVAL 40 +#define UDMA_READ_FUNC_CLEAR_FLAG_FAIL_WAIT 20 + +#define UDMA_HW_RST_TIMEOUT 1000 +#define UDMA_HW_RST_UNINT_DELAY 100 + +#define UDMA_HW_RST_COMPLETION_WAIT 20 + +#define UDMA_FUNC_IRQ_RSV 2 +#define UDMA_1US_CFG 999 +#define UDMA_EXT_LLM_ENTRY_SZ 8 +#define UDMA_EXT_LLM_MAX_DEPTH 4096 + +#define UDMA_BA_PG_SZ_SUPPORTED_256K 6 +#define UDMA_BA_PG_SZ_SUPPORTED_16K 2 + +#define UDMA_QUERY_PORT_INFO 1 +#define SPEED_100G 100000 +#define SPEED_200G 200000 + +#define UDMA_QUERY_COUNTER 8 +#define UDMA_QX_RESP 1 +#define UDMA_TX_RESP 3 +#define UDMA_TX_ERR_RESP 4 + +/* Fields of UDMA_OPC_EXT_CFG */ +#define EXT_CFG_VF_ID CMQ_REQ_FIELD_LOC(31, 0) +#define EXT_CFG_QP_PI_INDEX CMQ_REQ_FIELD_LOC(45, 32) +#define EXT_CFG_QP_PI_NUM CMQ_REQ_FIELD_LOC(63, 48) +#define EXT_CFG_QP_NUM CMQ_REQ_FIELD_LOC(87, 64) +#define EXT_CFG_QP_INDEX CMQ_REQ_FIELD_LOC(119, 96) +#define EXT_CFG_LLM_IDX CMQ_REQ_FIELD_LOC(139, 128) +#define EXT_CFG_LLM_NUM CMQ_REQ_FIELD_LOC(156, 144) + +#define CFG_LLM_A_BA_L CMQ_REQ_FIELD_LOC(31, 0) +#define CFG_LLM_A_BA_H CMQ_REQ_FIELD_LOC(63, 32) +#define CFG_LLM_A_DEPTH CMQ_REQ_FIELD_LOC(76, 64) +#define CFG_LLM_A_PG_SZ CMQ_REQ_FIELD_LOC(83, 80) +#define CFG_LLM_A_INIT_EN CMQ_REQ_FIELD_LOC(84, 84) +#define CFG_LLM_A_HEAD_BA_L CMQ_REQ_FIELD_LOC(127, 96) +#define CFG_LLM_A_HEAD_BA_H CMQ_REQ_FIELD_LOC(147, 128) +#define CFG_LLM_A_HEAD_NXT_PTR CMQ_REQ_FIELD_LOC(159, 148) +#define CFG_LLM_A_HEAD_PTR CMQ_REQ_FIELD_LOC(171, 160) +#define CFG_LLM_B_TAIL_BA_L CMQ_REQ_FIELD_LOC(31, 0) +#define CFG_LLM_B_TAIL_BA_H CMQ_REQ_FIELD_LOC(63, 32) +#define CFG_LLM_B_TAIL_PTR CMQ_REQ_FIELD_LOC(75, 64) + +/* + * Fields of UDMA_OPC_QUERY_PF_RES, UDMA_OPC_QUERY_VF_RES + * and UDMA_OPC_ALLOC_VF_RES + */ +#define FUNC_RES_A_VF_ID CMQ_REQ_FIELD_LOC(7, 0) +#define FUNC_RES_A_QPC_BT_INDEX CMQ_REQ_FIELD_LOC(42, 32) +#define FUNC_RES_A_QPC_BT_NUM CMQ_REQ_FIELD_LOC(59, 48) +#define FUNC_RES_A_SRQC_BT_INDEX CMQ_REQ_FIELD_LOC(72, 64) +#define FUNC_RES_A_SRQC_BT_NUM CMQ_REQ_FIELD_LOC(89, 80) +#define FUNC_RES_A_CQC_BT_INDEX CMQ_REQ_FIELD_LOC(104, 96) +#define FUNC_RES_A_CQC_BT_NUM CMQ_REQ_FIELD_LOC(121, 112) +#define FUNC_RES_A_MPT_BT_INDEX CMQ_REQ_FIELD_LOC(136, 128) +#define FUNC_RES_A_MPT_BT_NUM CMQ_REQ_FIELD_LOC(153, 144) +#define FUNC_RES_A_EQC_BT_INDEX CMQ_REQ_FIELD_LOC(168, 160) +#define FUNC_RES_A_EQC_BT_NUM CMQ_REQ_FIELD_LOC(185, 176) +#define FUNC_RES_B_SMAC_NUM CMQ_REQ_FIELD_LOC(48, 40) +#define FUNC_RES_B_SGID_NUM CMQ_REQ_FIELD_LOC(80, 72) +#define FUNC_RES_B_QID_INDEX CMQ_REQ_FIELD_LOC(105, 96) +#define FUNC_RES_B_QID_NUM CMQ_REQ_FIELD_LOC(122, 112) +#define FUNC_RES_V_QID_NUM CMQ_REQ_FIELD_LOC(115, 112) + +#define FUNC_RES_B_SCCC_BT_INDEX CMQ_REQ_FIELD_LOC(136, 128) +#define FUNC_RES_B_SCCC_BT_NUM CMQ_REQ_FIELD_LOC(145, 137) +#define FUNC_RES_B_GMV_BT_INDEX CMQ_REQ_FIELD_LOC(167, 160) +#define FUNC_RES_B_GMV_BT_NUM CMQ_REQ_FIELD_LOC(176, 168) +#define FUNC_RES_V_GMV_BT_NUM CMQ_REQ_FIELD_LOC(184, 176) + +/* Fields of UDMA_OPC_QUERY_PF_TIMER_RES */ +#define PF_TIMER_RES_QPC_ITEM_NUM CMQ_REQ_FIELD_LOC(60, 48) +#define PF_TIMER_RES_CQC_ITEM_NUM CMQ_REQ_FIELD_LOC(91, 80) + +/* Fields of UDMA_QUERY_RAM_ECC */ +#define QUERY_RAM_ECC_1BIT_ERR CMQ_REQ_FIELD_LOC(31, 0) +#define QUERY_RAM_ECC_RES_TYPE CMQ_REQ_FIELD_LOC(63, 32) +#define QUERY_RAM_ECC_TAG CMQ_REQ_FIELD_LOC(95, 64) + +enum { + UDMA_CMD_FLAG_IN = BIT(0), + UDMA_CMD_FLAG_NEXT = BIT(2), + UDMA_CMD_FLAG_WR = BIT(3), + UDMA_CMD_FLAG_NO_INTR = BIT(4), +}; + +enum udma_cmd_return_status { + CMD_EXEC_SUCCESS = 0, + CMD_NOT_EXIST = 2, +}; + +enum { + QP_ST_RST = 0, + QP_ST_ERR = 6, +}; + +/* CMQ command */ +enum udma_opcode_type { + UDMA_QUERY_FW_VER = 0x0001, + UDMA_OPC_CFG_DCQCN_PARAM = 0x1A80, + UDMA_OPC_CFG_LDCP_PARAM = 0x1A81, + UDMA_OPC_CFG_HC3_PARAM = 0x1A82, + UDMA_OPC_CFG_DIP_PARAM = 0x1A83, + UDMA_OPC_QUERY_HW_ID = 0x7032, + UDMA_OPC_QUERY_HW_VER = 0x8000, + UDMA_OPC_CFG_GLOBAL_PARAM = 0x8001, + UDMA_OPC_QUERY_PF_RES = 0x8400, + UDMA_OPC_ALLOC_VF_RES = 0x8401, + UDMA_OPC_CFG_EXT_LLM = 0x8403, + UDMA_OPC_QUERY_PF_TIMER_RES = 0x8406, + UDMA_OPC_QUERY_FUNC_INFO = 0x8407, + UDMA_OPC_QUERY_PF_CAPS_NUM = 0x8408, + UDMA_OPC_CFG_ENTRY_SIZE = 0x8409, + UDMA_OPC_QUERY_VF_CAPS_NUM = 0x8410, + UDMA_OPC_POST_MB = 0x8504, + UDMA_OPC_QUERY_MB_ST = 0x8505, + UDMA_OPC_CFG_BT_ATTR = 0x8506, + UDMA_OPC_FUNC_CLEAR = 0x8508, + UDMA_OPC_CLEAR_EXTDB_LIST_INFO = 0x850d, + UDMA_OPC_QUERY_VF_RES = 0x850e, + UDMA_OPC_CFG_GMV_BT = 0x8510, + UDMA_OPC_EXT_CFG = 0x8512, + UDMA_QUERY_RAM_ECC = 0x8513, + UDMA_SWITCH_PARAMETER_CFG = 0x1033, + UDMA_QUERY_OOR_CAPS = 0xA002, + UDMA_OPC_DEID_TBL_ADD = 0xA110, + UDMA_OPC_CFG_GMV_TBL = 0xA140, + UDMA_OPC_CFG_POE_ADDR = 0x801B, + UDMA_OPC_CFG_POE_ATTR = 0x801C, + UDMA_OPC_QUERY_COUNTER = 0x8206, + UDMA_OPC_QUERY_PORT_INFO = 0x7104, +}; + +#define UDMA_QUERY_PF_CAPS_CMD_NUM 5 +struct udma_query_pf_caps_a { + uint8_t number_ports; + uint8_t local_ca_ack_delay; + uint16_t max_sq_sg; + uint16_t max_sq_inline; + uint16_t max_rq_sg; + uint32_t max_extend_sg; + uint16_t num_qpc_timer; + uint16_t num_cqc_timer; + uint16_t max_srq_sges; + uint8_t num_aeq_vectors; + uint8_t num_other_vectors; + uint8_t max_sq_desc_sz; + uint8_t max_rq_desc_sz; + uint8_t max_srq_desc_sz; + uint8_t cqe_sz; +}; + +struct udma_query_pf_caps_b { + uint8_t mtpt_entry_sz; + uint8_t irrl_entry_sz; + uint8_t trrl_entry_sz; + uint8_t cqc_entry_sz; + uint8_t srqc_entry_sz; + uint8_t idx_entry_sz; + uint8_t sccc_sz; + uint8_t max_mtu; + uint16_t qpc_sz; + uint16_t qpc_timer_entry_sz; + uint16_t cqc_timer_entry_sz; + uint8_t min_cqes; + uint8_t min_wqes; + uint32_t page_size_cap; + uint8_t pkey_table_len; + uint8_t phy_num_uars; + uint8_t ctx_hop_num; + uint8_t pbl_hop_num; +}; + +struct udma_query_pf_caps_c { + uint32_t cap_flags_num_pds; + uint32_t max_gid_num_cqs; + uint32_t cq_depth; + uint32_t num_mrws; + uint32_t ord_num_qps; + uint16_t sq_depth; + uint16_t rq_depth; +}; + +struct udma_query_pf_caps_d { + uint32_t wq_hop_num_max_srqs; + uint16_t srq_depth; + uint16_t cap_flags_ex; + uint32_t num_ceqs_ceq_depth; + uint32_t arm_st_aeq_depth; + uint32_t num_uars_rsv_pds; + uint32_t rsv_uars_rsv_qps; +}; + +struct udma_query_pf_caps_e { + uint32_t chunk_size_shift_rsv_mrws; + uint32_t rsv_cqs; + uint32_t rsv_srqs; + uint32_t rsv_lkey; + uint16_t ceq_max_cnt; + uint16_t ceq_period; + uint16_t aeq_max_cnt; + uint16_t aeq_period; +}; + +#define UDMA_EXT_LLM_ENTRY(addr, id) (((id) << (64 - 12)) | ((addr) >> 12)) +#define UDMA_EXT_LLM_MIN_PAGES(que_num) ((que_num) * 4 + 2) + +#define QUERY_PF_CAPS_C_NUM_PDS_S 0 +#define QUERY_PF_CAPS_C_NUM_PDS_M GENMASK(19, 0) + +#define QUERY_PF_CAPS_C_CAP_FLAGS_S 20 +#define QUERY_PF_CAPS_C_CAP_FLAGS_M GENMASK(31, 20) + +#define QUERY_PF_CAPS_C_NUM_CQS_S 0 +#define QUERY_PF_CAPS_C_NUM_CQS_M GENMASK(19, 0) + +#define QUERY_PF_CAPS_C_MAX_GID_S 20 +#define QUERY_PF_CAPS_C_MAX_GID_M GENMASK(28, 20) + +#define QUERY_PF_CAPS_C_CQ_DEPTH_S 0 +#define QUERY_PF_CAPS_C_CQ_DEPTH_M GENMASK(22, 0) + +#define QUERY_PF_CAPS_C_NUM_MRWS_S 0 +#define QUERY_PF_CAPS_C_NUM_MRWS_M GENMASK(19, 0) + +#define QUERY_PF_CAPS_C_NUM_QPS_S 0 +#define QUERY_PF_CAPS_C_NUM_QPS_M GENMASK(19, 0) + +#define QUERY_PF_CAPS_C_MAX_ORD_S 20 +#define QUERY_PF_CAPS_C_MAX_ORD_M GENMASK(27, 20) + +#define QUERY_PF_CAPS_D_NUM_SRQS_S 0 +#define QUERY_PF_CAPS_D_NUM_SRQS_M GENMASK(19, 0) + +#define QUERY_PF_CAPS_D_RQWQE_HOP_NUM_S 20 +#define QUERY_PF_CAPS_D_RQWQE_HOP_NUM_M GENMASK(21, 20) + +#define QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_S 22 +#define QUERY_PF_CAPS_D_EX_SGE_HOP_NUM_M GENMASK(23, 22) + +#define QUERY_PF_CAPS_D_SQWQE_HOP_NUM_S 24 +#define QUERY_PF_CAPS_D_SQWQE_HOP_NUM_M GENMASK(25, 24) + +#define QUERY_PF_CAPS_D_CONG_TYPE_S 26 +#define QUERY_PF_CAPS_D_CONG_TYPE_M GENMASK(29, 26) + +/* default cap configuration for vf only */ +#define DEFAULT_AEQ_ARM_ST 0x3 +#define DEFAULT_CEQ_ARM_ST 0x3 +#define DEFAULT_CEQ_MAX_CNT 0x1 +#define DEFAULT_CEQ_PERIOD 0x10 +#define DEFAULT_AEQ_MAX_CNT 0x1 +#define DEFAULT_AEQ_PERIOD 0x10 + +#define QUERY_PF_CAPS_D_CEQ_DEPTH_S 0 +#define QUERY_PF_CAPS_D_CEQ_DEPTH_M GENMASK(21, 0) + +#define QUERY_PF_CAPS_D_NUM_CEQS_S 22 +#define QUERY_PF_CAPS_D_NUM_CEQS_M GENMASK(31, 22) + +#define QUERY_PF_CAPS_D_AEQ_DEPTH_S 0 +#define QUERY_PF_CAPS_D_AEQ_DEPTH_M GENMASK(21, 0) + +#define QUERY_PF_CAPS_D_AEQ_ARM_ST_S 22 +#define QUERY_PF_CAPS_D_AEQ_ARM_ST_M GENMASK(23, 22) + +#define QUERY_PF_CAPS_D_CEQ_ARM_ST_S 24 +#define QUERY_PF_CAPS_D_CEQ_ARM_ST_M GENMASK(25, 24) + +#define QUERY_PF_CAPS_D_RSV_PDS_S 0 +#define QUERY_PF_CAPS_D_RSV_PDS_M GENMASK(19, 0) + +#define QUERY_PF_CAPS_D_NUM_UARS_S 20 +#define QUERY_PF_CAPS_D_NUM_UARS_M GENMASK(27, 20) + +#define QUERY_PF_CAPS_D_RSV_QPS_S 0 +#define QUERY_PF_CAPS_D_RSV_QPS_M GENMASK(19, 0) + +#define QUERY_PF_CAPS_D_RSV_UARS_S 20 +#define QUERY_PF_CAPS_D_RSV_UARS_M GENMASK(27, 20) + +#define QUERY_PF_CAPS_E_RSV_MRWS_S 0 +#define QUERY_PF_CAPS_E_RSV_MRWS_M GENMASK(19, 0) + +#define QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_S 20 +#define QUERY_PF_CAPS_E_CHUNK_SIZE_SHIFT_M GENMASK(31, 20) + +#define QUERY_PF_CAPS_E_RSV_CQS_S 0 +#define QUERY_PF_CAPS_E_RSV_CQS_M GENMASK(19, 0) + +#define QUERY_PF_CAPS_E_RSV_SRQS_S 0 +#define QUERY_PF_CAPS_E_RSV_SRQS_M GENMASK(19, 0) + +#define QUERY_PF_CAPS_E_RSV_LKEYS_S 0 +#define QUERY_PF_CAPS_E_RSV_LKEYS_M GENMASK(19, 0) + +/* Fields of UDMA_OPC_CFG_BT_ATTR */ +#define CFG_BT_QPC_BA_PGSZ CMQ_REQ_FIELD_LOC(3, 0) +#define CFG_BT_QPC_BUF_PGSZ CMQ_REQ_FIELD_LOC(7, 4) +#define CFG_BT_QPC_HOPNUM CMQ_REQ_FIELD_LOC(9, 8) +#define CFG_BT_SRQC_BA_PGSZ CMQ_REQ_FIELD_LOC(35, 32) +#define CFG_BT_SRQC_BUF_PGSZ CMQ_REQ_FIELD_LOC(39, 36) +#define CFG_BT_SRQC_HOPNUM CMQ_REQ_FIELD_LOC(41, 40) +#define CFG_BT_CQC_BA_PGSZ CMQ_REQ_FIELD_LOC(67, 64) +#define CFG_BT_CQC_BUF_PGSZ CMQ_REQ_FIELD_LOC(71, 68) +#define CFG_BT_CQC_HOPNUM CMQ_REQ_FIELD_LOC(73, 72) +#define CFG_BT_MPT_BA_PGSZ CMQ_REQ_FIELD_LOC(99, 96) +#define CFG_BT_MPT_BUF_PGSZ CMQ_REQ_FIELD_LOC(103, 100) +#define CFG_BT_MPT_HOPNUM CMQ_REQ_FIELD_LOC(105, 104) +#define CFG_BT_SCCC_BA_PGSZ CMQ_REQ_FIELD_LOC(131, 128) +#define CFG_BT_SCCC_BUF_PGSZ CMQ_REQ_FIELD_LOC(135, 132) +#define CFG_BT_SCCC_HOPNUM CMQ_REQ_FIELD_LOC(137, 136) + +/* Fields of UDMA_OPC_CFG_ENTRY_SIZE */ +#define CFG_HEM_ENTRY_SIZE_TYPE CMQ_REQ_FIELD_LOC(31, 0) +enum { + UDMA_CFG_QPC_SIZE = BIT(0), + UDMA_CFG_SCCC_SIZE = BIT(1), +}; + +#define CFG_HEM_ENTRY_SIZE_VALUE CMQ_REQ_FIELD_LOC(191, 160) + +#define CFG_GMV_BT_BA_L CMQ_REQ_FIELD_LOC(31, 0) +#define CFG_GMV_BT_BA_H CMQ_REQ_FIELD_LOC(51, 32) +#define CFG_GMV_BT_IDX CMQ_REQ_FIELD_LOC(95, 64) +#define CFG_GMV_BT_VF_ID CMQ_REQ_FIELD_LOC(103, 96) + +#define UDMA_INT_NAME_LEN 32 + +struct udma_cfg_gmv_tb_a { + uint32_t vf_sgid_l; + uint32_t vf_sgid_ml; + uint32_t vf_sgid_mh; + uint32_t vf_sgid_h; + uint32_t vf_type_vlan_smac; + uint32_t vf_smac_h; +}; + +struct udma_cfg_gmv_tb_b { + uint32_t vf_upi; + uint32_t vf_eid_high; + uint32_t table_idx_rsv; + uint32_t vf_id; + uint32_t resv[2]; +}; + +#define SGID_TYPE_IPV4 1 + +#define CFG_GMV_TB_VF_SGID_TYPE_S 0 +#define CFG_GMV_TB_VF_SMAC_L_S 16 +#define CFG_GMV_TB_VF_PATTERN_S 3 +#define CFG_GMV_TB_VF_SGID_TYPE_M GENMASK(1, 0) +#define CFG_GMV_TB_VF_SMAC_L_M GENMASK(31, 16) + +#define SGID_H_SHIFT 12 +#define SMAC_L_SHIFT 0 +#define SMAC_H_SHIFT 2 + +#define CFG_GMV_TBL_CMD_NUM 2 + +struct udma_eid_tbl_entry_cmd { + uint8_t resp_code; + uint8_t rsv1[3]; + uint32_t eid_addr[4]; + uint16_t eid_ad; + uint8_t rsv2[2]; +}; + +#define UDMA_EID_TB_VFID_S 0 +#define UDMA_EID_TB_VFID_M GENMASK(7, 0) +#define UDMA_EID_TB_RES_SUCCESS 0 +#define UDMA_EID_TB_RES_MODIFY 2 + +union udma_eid { + union ubcore_eid ubcore_eid; + struct { + uint32_t eid_l; + uint32_t eid_ml; + uint32_t eid_mh; + uint32_t eid_h; + } bit32_data; +}; + +struct udma_poe_cfg_addr_cmq { + uint32_t channel_id; + uint32_t poe_addr_l; + uint32_t poe_addr_h; + uint32_t rsv[3]; +}; + +struct udma_poe_cfg_attr_cmq { + uint32_t channel_id; + uint32_t rsv_en_outstd; + uint32_t rsv[4]; +}; + +struct udma_port_info_cmq { + uint32_t speed; + uint8_t query_type; + uint8_t rsv[19]; +}; + +struct udma_rx_cnt_cmd_data { + uint64_t rsv; + uint64_t pkt_rx_cnt; + uint64_t err_pkt_rx_cnt; +}; + +struct udma_tx_cnt_cmd_data { + uint64_t rsv[2]; + uint64_t pkt_tx_cnt; +}; + +struct udma_tx_err_cnt_cmd_data { + uint64_t err_pkt_tx_cnt; + uint64_t rsv[2]; +}; + +#endif /* _UDMA_HW_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_jetty.c b/drivers/ub/hw/hns3/hns3_udma_jetty.c new file mode 100644 index 0000000000000000000000000000000000000000..31d40200c32cbd57ea0d5d4d92b2bb84387cafec --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_jetty.c @@ -0,0 +1,494 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include +#include "urma/ubcore_types.h" +#include "hns3_udma_tp.h" +#include "hns3_udma_jfc.h" +#include "hns3_udma_jfr.h" +#include "hns3_udma_db.h" +#include "hns3_udma_dfx.h" +#include "hns3_udma_jetty.h" + +static void init_jetty_cfg(struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg) +{ + jetty->shared_jfr = cfg->flag.bs.share_jfr; + jetty->tp_mode = cfg->trans_mode; + jetty->ubcore_jetty.jetty_cfg = *cfg; +} + +static void udma_fill_jetty_um_qp_attr(struct udma_dev *dev, + struct udma_jetty *jetty, + struct ubcore_ucontext *uctx, + const struct ubcore_jetty_cfg *cfg) +{ + struct udma_ucontext *udma_ctx = to_udma_ucontext(uctx); + struct udma_qp_attr *qp_attr = &jetty->qp.qp_attr; + + qp_attr->is_jetty = true; + qp_attr->uctx = uctx; + qp_attr->pdn = udma_ctx->pdn; + qp_attr->send_jfc = to_udma_jfc(cfg->send_jfc); + qp_attr->jetty = jetty; + qp_attr->cap.max_send_wr = cfg->jfs_depth; + qp_attr->cap.max_send_sge = cfg->max_send_sge; + qp_attr->cap.max_inline_data = cfg->max_inline_data; + qp_attr->cap.retry_cnt = cfg->retry_cnt; + qp_attr->cap.rnr_retry = cfg->rnr_retry; + qp_attr->cap.ack_timeout = cfg->err_timeout; + qp_attr->qp_type = QPT_UD; + + qp_attr->jfr = jetty->udma_jfr; + qp_attr->qpn_map = &jetty->qpn_map; + qp_attr->recv_jfc = to_udma_jfc(cfg->recv_jfc); + if (jetty->ubcore_jetty.jetty_cfg.priority >= dev->caps.sl_num) { + qp_attr->priority = + dev->caps.sl_num > 0 ? dev->caps.sl_num - 1 : 0; + dev_err(dev->dev, "The setted priority (%d) should smaller than the max priority (%d), priority (%d) is used\n", + jetty->ubcore_jetty.jetty_cfg.priority, + dev->caps.sl_num, qp_attr->priority); + } else { + qp_attr->priority = jetty->ubcore_jetty.jetty_cfg.priority; + } +} + +static int udma_modify_qp_jetty(struct udma_dev *dev, struct udma_jetty *jetty, + enum udma_qp_state target_state) +{ + union ubcore_tp_attr_mask ubcore_attr_mask; + struct udma_modify_tp_attr m_attr = {}; + struct udma_qp *qp; + int ret; + + qp = &jetty->qp; + qp->udma_device = dev; + qp->send_jfc = qp->qp_attr.send_jfc; + qp->recv_jfc = qp->qp_attr.recv_jfc; + + m_attr.path_mtu = UBCORE_MTU_4096; + m_attr.hop_limit = MAX_HOP_LIMIT; + ubcore_attr_mask.value = 0; + qp->m_attr = &m_attr; + + ret = udma_modify_qp_common(qp, NULL, ubcore_attr_mask, jetty->qp.state, target_state); + if (ret) + dev_err(dev->dev, "failed to modify qpc to RTS in Jetty.\n"); + + qp->state = target_state; + return ret; +} + +static int set_jetty_jfr(struct udma_dev *dev, struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg, uint32_t jfr_id) +{ + if (cfg->jfr) { + jetty->shared_jfr = true; + jetty->udma_jfr = to_udma_jfr(cfg->jfr); + } else { + jetty->shared_jfr = false; + jetty->udma_jfr = get_udma_jfr(&dev->ub_dev, jfr_id); + if (!jetty->udma_jfr) { + dev_err(dev->dev, + "failed to find jfr, jfr_id:%u.\n", jfr_id); + return -EINVAL; + } + } + + return 0; +} + +static int alloc_jetty_um_qp(struct udma_dev *dev, struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg, + struct ubcore_udata *udata) +{ + int ret; + + udma_fill_jetty_um_qp_attr(dev, jetty, udata->uctx, cfg); + + ret = udma_create_qp_common(dev, &jetty->qp, udata); + if (ret) { + dev_err(dev->dev, "failed create qp for um jetty.\n"); + return ret; + } + + jetty->qp.state = QPS_RESET; + ret = udma_modify_qp_jetty(dev, jetty, QPS_RTS); + if (ret) + udma_destroy_qp_common(dev, &jetty->qp); + + return ret; +} + +static void set_jetty_ext_sge_param(struct udma_jetty *jetty) +{ + struct ubcore_jetty_cfg *jetty_cfg = &jetty->ubcore_jetty.jetty_cfg; + uint32_t max_inline_data; + uint32_t wqe_sge_cnt = 0; + uint32_t total_sge_cnt; + uint32_t ext_sge_cnt; + uint32_t sq_wqe_cnt; + uint32_t max_gs; + + sq_wqe_cnt = jetty->rc_node.wqe_cnt; + jetty->rc_node.sge_shift = UDMA_SGE_SHIFT; + max_inline_data = roundup_pow_of_two(jetty_cfg->max_inline_data); + ext_sge_cnt = max_inline_data / UDMA_SGE_SIZE; + + max_gs = max_t(uint32_t, ext_sge_cnt, jetty_cfg->max_send_sge); + if (max_gs > UDMA_SGE_IN_WQE) + wqe_sge_cnt = max_gs - UDMA_SGE_IN_WQE; + + if (wqe_sge_cnt) { + total_sge_cnt = roundup_pow_of_two(sq_wqe_cnt * wqe_sge_cnt); + jetty->rc_node.sge_cnt = max(total_sge_cnt, + (uint32_t)UDMA_PAGE_SIZE / + UDMA_SGE_SIZE); + } +} + +static int set_jetty_buf_attr(struct udma_dev *udma_dev, + struct udma_jetty *jetty, + struct udma_buf_attr *buf_attr) +{ + int totle_buff_size = 0; + uint32_t cfg_depth; + int buf_size; + int idx = 0; + + /* SQ WQE */ + jetty->rc_node.sge_offset = 0; + cfg_depth = roundup_pow_of_two(jetty->ubcore_jetty.jetty_cfg.jfs_depth); + jetty->rc_node.wqe_cnt = cfg_depth < UDMA_MIN_JFS_DEPTH ? + UDMA_MIN_JFS_DEPTH : cfg_depth; + jetty->rc_node.wqe_shift = UDMA_SQ_WQE_SHIFT; + set_jetty_ext_sge_param(jetty); + + buf_size = to_udma_hem_entries_size(jetty->rc_node.wqe_cnt, + jetty->rc_node.wqe_shift); + if (buf_size > 0) { + buf_attr->region[idx].size = buf_size; + buf_attr->region[idx].hopnum = udma_dev->caps.wqe_sq_hop_num; + idx++; + totle_buff_size += buf_size; + } + /* extend SGE WQE in SQ */ + jetty->rc_node.sge_offset = totle_buff_size; + + buf_size = to_udma_hem_entries_size(jetty->rc_node.sge_cnt, + jetty->rc_node.sge_shift); + if (buf_size > 0) { + buf_attr->region[idx].size = buf_size; + buf_attr->region[idx].hopnum = udma_dev->caps.wqe_sge_hop_num; + idx++; + totle_buff_size += buf_size; + } + + if (totle_buff_size < 1) + return -EINVAL; + + buf_attr->region_count = idx; + buf_attr->mtt_only = false; + buf_attr->page_shift = PAGE_SHIFT + udma_dev->caps.mtt_buf_pg_sz; + + return 0; +} + +static int alloc_jetty_buf(struct udma_dev *dev, struct udma_jetty *jetty, + const struct ubcore_jetty_cfg *cfg, struct ubcore_udata *udata) +{ + struct udma_create_jetty_ucmd ucmd = {}; + struct udma_buf_attr buf_attr = {}; + int ret; + + if (udata) { + ret = copy_from_user(&ucmd, (void *)udata->udrv_data->in_addr, + min(udata->udrv_data->in_len, + (uint32_t)sizeof(ucmd))); + if (ret) { + dev_err(dev->dev, + "failed to copy jetty udata, ret = %d.\n", + ret); + return -EFAULT; + } + } + + ret = set_jetty_jfr(dev, jetty, cfg, ucmd.jfr_id); + if (ret) + return ret; + + if (cfg->trans_mode == UBCORE_TP_UM) { + ret = alloc_jetty_um_qp(dev, jetty, cfg, udata); + if (ret) + return ret; + } else if (cfg->trans_mode == UBCORE_TP_RM) { + xa_init(&jetty->srm_node_table); + } else if (cfg->trans_mode == UBCORE_TP_RC) { + jetty->rc_node.buf_addr = ucmd.buf_addr; + if (!ucmd.buf_addr) { + jetty->dca_en = true; + return 0; + } + + ret = set_jetty_buf_attr(dev, jetty, &buf_attr); + if (ret) { + dev_err(dev->dev, + "failed to set jetty buf attr, ret = %d.\n", + ret); + return ret; + } + + ret = udma_mtr_create(dev, &jetty->rc_node.mtr, &buf_attr, + PAGE_SHIFT + dev->caps.mtt_ba_pg_sz, + ucmd.buf_addr, !!udata); + if (ret) { + dev_err(dev->dev, + "failed to create WQE mtr for RC Jetty, ret = %d.\n", + ret); + return ret; + } + + ret = udma_db_map_user(dev, ucmd.sdb_addr, &jetty->rc_node.sdb); + if (ret) { + dev_err(dev->dev, + "failed to map user sdb_addr, ret = %d.\n", + ret); + udma_mtr_destroy(dev, &jetty->rc_node.mtr); + return ret; + } + } + + return 0; +} + +static int alloc_jetty_id(struct udma_dev *udma_dev, struct udma_jetty *jetty) +{ + struct udma_jetty_table *jetty_table = &udma_dev->jetty_table; + struct udma_ida *jetty_ida = &jetty_table->jetty_ida; + int ret; + int id; + + id = ida_alloc_range(&jetty_ida->ida, jetty_ida->min, jetty_ida->max, + GFP_KERNEL); + if (id < 0) { + dev_err(udma_dev->dev, "failed to alloc jetty_id(%d).\n", id); + return id; + } + jetty->jetty_id = (uint32_t)id; + jetty->ubcore_jetty.id = jetty->jetty_id; + + ret = xa_err(xa_store(&jetty_table->xa, jetty->jetty_id, jetty, + GFP_KERNEL)); + if (ret) { + dev_err(udma_dev->dev, "failed to store Jetty, ret = %d.\n", + ret); + ida_free(&jetty_ida->ida, id); + } + + return ret; +} + +static void store_jetty_id(struct udma_dev *udma_dev, struct udma_jetty *jetty) +{ + struct jetty_list *jetty_new; + struct jetty_list *jetty_now; + unsigned long flags; + spinlock_t *lock; + int ret; + int i; + + ret = udma_find_dfx_dev(udma_dev, &i); + if (ret) + return; + + jetty_new = kzalloc(sizeof(struct jetty_list), GFP_KERNEL); + if (jetty_new == NULL) + return; + + lock = &g_udma_dfx_list[i].dfx->jetty_list->node_lock; + spin_lock_irqsave(lock, flags); + list_for_each_entry(jetty_now, + &g_udma_dfx_list[i].dfx->jetty_list->node, node) { + if (jetty_now->jetty_id == jetty->jetty_id) { + jetty_now->jfs_depth = + jetty->ubcore_jetty.jetty_cfg.jfs_depth; + jetty_now->jfr_depth = + jetty->ubcore_jetty.jetty_cfg.jfr_depth; + jetty_now->pri = + jetty->ubcore_jetty.jetty_cfg.priority; + jetty_now->jfr_id = jetty->udma_jfr->jfrn; + jetty_now->jfc_s_id = jetty->ubcore_jetty.jetty_cfg.send_jfc->id; + jetty_now->jfc_r_id = jetty->ubcore_jetty.jetty_cfg.recv_jfc->id; + goto found; + } + } + + jetty_new->jetty_id = jetty->jetty_id; + jetty_new->jfs_depth = jetty->ubcore_jetty.jetty_cfg.jfs_depth; + jetty_new->jfr_depth = jetty->ubcore_jetty.jetty_cfg.jfr_depth; + jetty_new->pri = jetty->ubcore_jetty.jetty_cfg.priority; + jetty_new->jfr_id = jetty->udma_jfr->jfrn; + jetty_new->jfc_s_id = jetty->ubcore_jetty.jetty_cfg.send_jfc->id; + jetty_new->jfc_r_id = jetty->ubcore_jetty.jetty_cfg.recv_jfc->id; + list_add(&jetty_new->node, &g_udma_dfx_list[i].dfx->jetty_list->node); + spin_unlock_irqrestore(lock, flags); + + return; + +found: + spin_unlock_irqrestore(lock, flags); + kfree(jetty_new); +} + +static void delete_jetty_id(struct udma_dev *udma_dev, + struct udma_jetty *jetty) +{ + struct jetty_list *jetty_now, *jetty_tmp; + unsigned long flags; + spinlock_t *lock; + int ret; + int i; + + ret = udma_find_dfx_dev(udma_dev, &i); + if (ret) + return; + + lock = &g_udma_dfx_list[i].dfx->jetty_list->node_lock; + spin_lock_irqsave(lock, flags); + list_for_each_entry_safe(jetty_now, jetty_tmp, + &g_udma_dfx_list[i].dfx->jetty_list->node, + node) { + if (jetty_now->jetty_id == jetty->jetty_id) { + list_del(&jetty_now->node); + kfree(jetty_now); + spin_unlock_irqrestore(lock, flags); + return; + } + } + spin_unlock_irqrestore(lock, flags); +} + +static void free_jetty_id(struct udma_dev *udma_dev, struct udma_jetty *jetty) +{ + struct udma_jetty_table *jetty_table = &udma_dev->jetty_table; + struct udma_ida *jetty_ida = &jetty_table->jetty_ida; + + xa_erase(&jetty_table->xa, jetty->jetty_id); + ida_free(&jetty_ida->ida, (int)jetty->jetty_id); +} + +struct ubcore_jetty *udma_create_jetty(struct ubcore_device *dev, + const struct ubcore_jetty_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + struct udma_jetty *jetty; + int ret; + + jetty = kzalloc(sizeof(struct udma_jetty), GFP_KERNEL); + if (!jetty) + return NULL; + + init_jetty_cfg(jetty, cfg); + ret = alloc_jetty_id(udma_dev, jetty); + if (ret) + goto err_alloc_jetty_id; + + init_jetty_x_qpn_bitmap(udma_dev, &jetty->qpn_map, + udma_dev->caps.num_jetty_shift, + UDMA_JETTY_QPN_PREFIX, jetty->jetty_id); + + ret = alloc_jetty_buf(udma_dev, jetty, cfg, udata); + if (ret) { + dev_err(udma_dev->dev, "alloc Jetty buf failed.\n"); + goto err_alloc_jetty_buf; + } + + mutex_init(&jetty->tp_mutex); + + if (dfx_switch) + store_jetty_id(udma_dev, jetty); + + return &jetty->ubcore_jetty; + +err_alloc_jetty_buf: + clean_jetty_x_qpn_bitmap(&jetty->qpn_map); + free_jetty_id(udma_dev, jetty); +err_alloc_jetty_id: + kfree(jetty); + + return NULL; +} + +int free_jetty_buf(struct udma_dev *dev, struct udma_jetty *jetty) +{ + int ret = 0; + + if (jetty->tp_mode == UBCORE_TP_UM) { + ret = udma_modify_qp_jetty(dev, jetty, QPS_RESET); + if (ret) + dev_err(dev->dev, + "modify qp(0x%llx) to RESET failed for um jetty.\n", + jetty->qp.qpn); + + udma_destroy_qp_common(dev, &jetty->qp); + } else if (jetty->tp_mode == UBCORE_TP_RC && !jetty->dca_en) { + udma_db_unmap_user(dev, &jetty->rc_node.sdb); + udma_mtr_destroy(dev, &jetty->rc_node.mtr); + } + + return ret; +} + +int udma_destroy_jetty(struct ubcore_jetty *jetty) +{ + struct udma_jetty *udma_jetty; + struct udma_dev *udma_dev; + int ret; + + udma_jetty = to_udma_jetty(jetty); + udma_dev = to_udma_dev(jetty->ub_dev); + + ret = free_jetty_buf(udma_dev, udma_jetty); + clean_jetty_x_qpn_bitmap(&udma_jetty->qpn_map); + + if (dfx_switch) + delete_jetty_id(udma_dev, udma_jetty); + + free_jetty_id(udma_dev, udma_jetty); + kfree(udma_jetty); + + return ret; +} + +struct ubcore_tjetty *udma_import_jetty(struct ubcore_device *dev, + const struct ubcore_tjetty_cfg *cfg, + struct ubcore_udata *udata) +{ + struct ubcore_tjetty *tjetty; + + tjetty = kcalloc(1, sizeof(struct ubcore_tjetty), GFP_KERNEL); + if (!tjetty) + return NULL; + + return tjetty; +} + +int udma_unimport_jetty(struct ubcore_tjetty *tjetty) +{ + kfree(tjetty); + + return 0; +} diff --git a/drivers/ub/hw/hns3/hns3_udma_jetty.h b/drivers/ub/hw/hns3/hns3_udma_jetty.h new file mode 100644 index 0000000000000000000000000000000000000000..25c8b6e3bc54ca02cc7d697391060d158aeacafd --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_jetty.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_JETTY_H +#define _UDMA_JETTY_H + +#include "hns3_udma_qp.h" + +struct rc_node { + struct udma_tp *tp; + uint32_t tpn; + uint64_t buf_addr; + struct udma_mtr mtr; + uint32_t wqe_cnt; + uint32_t wqe_shift; + uint32_t sge_offset; + uint32_t sge_cnt; + uint32_t sge_shift; + struct udma_db sdb; + struct ubcore_jetty_id tjetty_id; +}; + +struct udma_jetty { + struct ubcore_jetty ubcore_jetty; + bool shared_jfr; + struct udma_jfr *udma_jfr; + enum ubcore_transport_mode tp_mode; + union { + struct rc_node rc_node; + struct xarray srm_node_table; + struct udma_qp qp; + }; + struct udma_qpn_bitmap qpn_map; + uint32_t jetty_id; + struct mutex tp_mutex; + bool dca_en; +}; + +static inline struct udma_jetty *to_udma_jetty(struct ubcore_jetty *ubcore_jetty) +{ + return container_of(ubcore_jetty, struct udma_jetty, ubcore_jetty); +} + +struct ubcore_jetty *udma_create_jetty(struct ubcore_device *dev, + const struct ubcore_jetty_cfg *cfg, + struct ubcore_udata *udata); +int udma_destroy_jetty(struct ubcore_jetty *jetty); +struct ubcore_tjetty *udma_import_jetty(struct ubcore_device *dev, + const struct ubcore_tjetty_cfg *cfg, + struct ubcore_udata *udata); +int udma_unimport_jetty(struct ubcore_tjetty *tjetty); + +#endif /* _UDMA_JETTY_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_jfc.c b/drivers/ub/hw/hns3/hns3_udma_jfc.c new file mode 100644 index 0000000000000000000000000000000000000000..4abb9ad2e84f3661836377fd95eef9b04b43ba9f --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_jfc.c @@ -0,0 +1,737 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include +#include "hns3_udma_hem.h" +#include "hns3_udma_cmd.h" +#include "hns3_udma_db.h" +#include "hns3_udma_dfx.h" +#include "hns3_udma_jfc.h" + +static int udma_hw_create_cq(struct udma_dev *dev, + struct udma_cmd_mailbox *mailbox, uint32_t cqn) +{ + struct udma_cmq_desc desc; + struct udma_mbox *mb; + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mbox_desc_init(mb, mailbox->dma, 0, cqn, UDMA_CMD_CREATE_CQC); + + return udma_cmd_mbox(dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); +} + +static int udma_hw_destroy_cq(struct udma_dev *dev, uint32_t cqn) +{ + struct udma_cmq_desc desc; + struct udma_mbox *mb; + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mbox_desc_init(mb, 0, 0, cqn, UDMA_CMD_DESTROY_CQC); + + return udma_cmd_mbox(dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); +} + +static int check_jfc_cfg(struct udma_dev *udma_dev, const struct ubcore_jfc_cfg *cfg) +{ + if (!cfg->depth || cfg->depth > udma_dev->caps.max_cqes) { + dev_err(udma_dev->dev, + "failed to check jfc attr depth = 0x%x.\n", + cfg->depth); + return -EINVAL; + } + + return 0; +} + +static int check_poe_attr(struct udma_dev *udma_dev, + struct udma_jfc_attr_ex *jfc_attr_ex) +{ + if (!(udma_dev->caps.flags & UDMA_CAP_FLAG_POE)) { + dev_err(udma_dev->dev, "Unsupport POE JFC.\n"); + return -EINVAL; + } + + return 0; +} + +static int check_notify_attr(struct udma_dev *udma_dev, + struct udma_jfc_attr_ex *jfc_attr_ex) +{ + if (!(udma_dev->caps.flags & UDMA_CAP_FLAG_WRITE_NOTIFY)) { + dev_err(udma_dev->dev, "Unsupport NOTIFY JFC.\n"); + return -EINVAL; + } + + switch (jfc_attr_ex->notify_mode) { + case UDMA_JFC_NOTIFY_MODE_4B_ALIGN: + case UDMA_JFC_NOTIFY_MODE_DDR_4B_ALIGN: + break; + case UDMA_JFC_NOTIFY_MODE_64B_ALIGN: + case UDMA_JFC_NOTIFY_MODE_DDR_64B_ALIGN: + dev_err(udma_dev->dev, "Doesn't support notify mode %u\n", + jfc_attr_ex->notify_mode); + return -EINVAL; + default: + dev_err(udma_dev->dev, "Invalid notify mode %u\n", + jfc_attr_ex->notify_mode); + return -EINVAL; + } + + if (jfc_attr_ex->notify_addr & UDMA_ADDR_4K_MASK) { + dev_err(udma_dev->dev, + "Notify addr should be aligned to 4k.\n"); + return -EINVAL; + } + + return 0; +} + +static int check_jfc_attr_ex(struct udma_dev *udma_dev, + struct udma_jfc_attr_ex *jfc_attr_ex) +{ + int ret; + + switch (jfc_attr_ex->create_flags) { + case UDMA_JFC_CREATE_ENABLE_POE_MODE: + ret = check_poe_attr(udma_dev, jfc_attr_ex); + break; + case UDMA_JFC_CREATE_ENABLE_NOTIFY: + ret = check_notify_attr(udma_dev, jfc_attr_ex); + break; + default: + dev_err(udma_dev->dev, "Invalid create flags %llu\n", + jfc_attr_ex->create_flags); + return -EINVAL; + } + + return ret; +} + +static int check_create_jfc(struct udma_dev *udma_dev, + const struct ubcore_jfc_cfg *cfg, + struct udma_create_jfc_ucmd *ucmd, + struct ubcore_udata *udata) +{ + int ret; + + if (udata) { + ret = copy_from_user((void *)ucmd, + (void *)udata->udrv_data->in_addr, + min(udata->udrv_data->in_len, + (uint32_t)sizeof(struct udma_create_jfc_ucmd))); + if (ret) { + dev_err(udma_dev->dev, + "failed to copy JFC udata, ret = %d.\n", ret); + return ret; + } + } + + ret = check_jfc_cfg(udma_dev, cfg); + if (ret) { + dev_err(udma_dev->dev, "failed to check JFC cfg.\n"); + return ret; + } + + if (ucmd->jfc_attr_ex.jfc_ex_mask & + UDMA_JFC_NOTIFY_OR_POE_CREATE_FLAGS) { + ret = check_jfc_attr_ex(udma_dev, &ucmd->jfc_attr_ex); + if (ret) { + dev_err(udma_dev->dev, + "failed to check JFC attr ex.\n"); + return ret; + } + } + + return 0; +} + +void set_jfc_param(struct udma_jfc *udma_jfc, const struct ubcore_jfc_cfg *cfg) +{ + udma_jfc->jfc_depth = roundup_pow_of_two(cfg->depth); + memcpy(&udma_jfc->ubcore_jfc.jfc_cfg, cfg, sizeof(struct ubcore_jfc_cfg)); +} + +static void init_jfc(struct udma_jfc *udma_jfc, struct udma_create_jfc_ucmd *ucmd) +{ + spin_lock_init(&udma_jfc->lock); + INIT_LIST_HEAD(&udma_jfc->sq_list); + INIT_LIST_HEAD(&udma_jfc->rq_list); + if (ucmd->jfc_attr_ex.jfc_ex_mask & UDMA_JFC_NOTIFY_OR_POE_CREATE_FLAGS) + udma_jfc->jfc_attr_ex = ucmd->jfc_attr_ex; +} + +static int alloc_jfc_cqe_buf(struct udma_dev *dev, struct udma_jfc *jfc, + struct ubcore_udata *udata, uint64_t addr) +{ + struct udma_buf_attr buf_attr = {}; + int ret; + + buf_attr.page_shift = dev->caps.cqe_buf_pg_sz + PAGE_SHIFT; + buf_attr.region[0].size = jfc->jfc_depth * dev->caps.cqe_sz; + buf_attr.region[0].hopnum = dev->caps.cqe_hop_num; + buf_attr.region_count = 1; + ret = udma_mtr_create(dev, &jfc->mtr, &buf_attr, + dev->caps.cqe_ba_pg_sz + PAGE_SHIFT, + addr, udata ? true : false); + if (ret) + dev_err(dev->dev, + "failed to alloc JFC buf, ret = %d.\n", ret); + + return ret; +} + +static void free_jfc_cqe_buf(struct udma_dev *dev, struct udma_jfc *jfc) +{ + udma_mtr_destroy(dev, &jfc->mtr); +} + +static int alloc_jfc_buf(struct udma_dev *udma_dev, struct udma_jfc *udma_jfc, + struct ubcore_udata *udata, + struct udma_create_jfc_ucmd *ucmd) +{ + struct udma_create_jfc_resp resp = {}; + int ret; + + ret = alloc_jfc_cqe_buf(udma_dev, udma_jfc, udata, ucmd->buf_addr); + if (ret) + return ret; + + if (udma_dev->caps.flags & UDMA_CAP_FLAG_CQ_RECORD_DB) { + ret = udma_db_map_user(udma_dev, ucmd->db_addr, &udma_jfc->db); + if (ret) { + dev_err(udma_dev->dev, + "failed to map JFC db, ret = %d.\n", ret); + goto db_err; + } + udma_jfc->jfc_caps |= UDMA_JFC_CAP_RECORD_DB; + } + + if (udata) { + resp.jfc_caps = udma_jfc->jfc_caps; + ret = copy_to_user((void *)udata->udrv_data->out_addr, &resp, + min(udata->udrv_data->out_len, + (uint32_t)sizeof(resp))); + if (ret) { + dev_err(udma_dev->dev, + "failed to copy jfc resp, ret = %d\n", ret); + goto err_copy; + } + } + refcount_set(&udma_jfc->refcount, 1); + init_completion(&udma_jfc->free); + return ret; + +err_copy: + if (udma_dev->caps.flags & UDMA_CAP_FLAG_CQ_RECORD_DB) { + udma_db_unmap_user(udma_dev, &udma_jfc->db); + udma_jfc->jfc_caps &= ~UDMA_JFC_CAP_RECORD_DB; + } +db_err: + free_jfc_cqe_buf(udma_dev, udma_jfc); + + return ret; +} + +static void set_write_notify_param(struct udma_dev *udma_dev, + struct udma_jfc *udma_jfc, + struct udma_jfc_context *jfc_context) +{ + uint8_t device_mode; + + if (udma_jfc->jfc_attr_ex.notify_mode == UDMA_JFC_NOTIFY_MODE_4B_ALIGN) + device_mode = UDMA_NOTIFY_DEV; + else + device_mode = UDMA_NOTIFY_DDR; + + udma_reg_enable(jfc_context, CQC_NOTIFY_EN); + udma_reg_write(jfc_context, CQC_NOTIFY_DEVICE_EN, device_mode); + /* Supports only 4B alignment. */ + udma_reg_write(jfc_context, CQC_NOTIFY_MODE, UDMA_NOTIFY_MODE_4B); + udma_reg_write(jfc_context, CQC_NOTIFY_ADDR_0, + (uint32_t)udma_get_field64(udma_jfc->jfc_attr_ex.notify_addr, + CQC_NOTIFY_ADDR_0_M, CQC_NOTIFY_ADDR_0_S)); + udma_reg_write(jfc_context, CQC_POE_QID, + (uint32_t)udma_get_field64(udma_jfc->jfc_attr_ex.notify_addr, + CQC_NOTIFY_ADDR_1_M, CQC_NOTIFY_ADDR_1_S)); + udma_reg_write(jfc_context, CQC_NOTIFY_ADDR_2, + (uint32_t)udma_get_field64(udma_jfc->jfc_attr_ex.notify_addr, + CQC_NOTIFY_ADDR_2_M, CQC_NOTIFY_ADDR_2_S)); + udma_reg_write(jfc_context, CQC_NOTIFY_ADDR_3, + (uint32_t)udma_get_field64(udma_jfc->jfc_attr_ex.notify_addr, + CQC_NOTIFY_ADDR_3_M, CQC_NOTIFY_ADDR_3_S)); + udma_reg_write(jfc_context, CQC_NOTIFY_ADDR_4, + (uint32_t)udma_get_field64(udma_jfc->jfc_attr_ex.notify_addr, + CQC_NOTIFY_ADDR_4_M, CQC_NOTIFY_ADDR_4_S)); + udma_reg_write(jfc_context, CQC_NOTIFY_ADDR_5, + (uint32_t)udma_get_field64(udma_jfc->jfc_attr_ex.notify_addr, + CQC_NOTIFY_ADDR_5_M, CQC_NOTIFY_ADDR_5_S)); + udma_reg_write(jfc_context, CQC_NOTIFY_ADDR_6, + (uint32_t)udma_get_field64(udma_jfc->jfc_attr_ex.notify_addr, + CQC_NOTIFY_ADDR_6_M, CQC_NOTIFY_ADDR_6_S)); +} + +static void udma_write_jfc_cqc(struct udma_dev *udma_dev, struct udma_jfc *udma_jfc, + void *mb_buf, uint64_t *mtts, uint64_t dma_handle) +{ + struct udma_jfc_context *jfc_context; + + jfc_context = (struct udma_jfc_context *)mb_buf; + memset(jfc_context, 0, sizeof(*jfc_context)); + + udma_reg_write(jfc_context, CQC_CQ_ST, CQ_STATE_VALID); + udma_reg_write(jfc_context, CQC_ARM_ST, NO_ARMED); + + udma_reg_write(jfc_context, CQC_SHIFT, ilog2(udma_jfc->jfc_depth)); + udma_reg_write(jfc_context, CQC_CQN, udma_jfc->cqn); + + udma_reg_write(jfc_context, CQC_CQE_SIZE, CQE_SIZE_64B); + + udma_reg_write(jfc_context, CQC_CQE_CUR_BLK_ADDR_L, + to_hr_hw_page_addr(mtts[0])); + udma_reg_write(jfc_context, CQC_CQE_CUR_BLK_ADDR_H, + upper_32_bits(to_hr_hw_page_addr(mtts[0]))); + udma_reg_write(jfc_context, CQC_CQE_HOP_NUM, + udma_dev->caps.cqe_hop_num == + UDMA_HOP_NUM_0 ? 0 : udma_dev->caps.cqe_hop_num); + udma_reg_write(jfc_context, CQC_CQE_NEX_BLK_ADDR_L, + to_hr_hw_page_addr(mtts[1])); + udma_reg_write(jfc_context, CQC_CQE_NEX_BLK_ADDR_H, + upper_32_bits(to_hr_hw_page_addr(mtts[1]))); + udma_reg_write(jfc_context, CQC_CQE_BAR_PG_SZ, + to_hr_hw_page_shift(udma_jfc->mtr.hem_cfg.ba_pg_shift)); + udma_reg_write(jfc_context, CQC_CQE_BUF_PG_SZ, + to_hr_hw_page_shift(udma_jfc->mtr.hem_cfg.buf_pg_shift)); + udma_reg_write(jfc_context, CQC_CQE_BA_L, + dma_handle >> CQC_CQE_BA_L_OFFSET); + udma_reg_write(jfc_context, CQC_CQE_BA_H, + dma_handle >> CQC_CQE_BA_H_OFFSET); + udma_reg_write(jfc_context, CQC_CQ_MAX_CNT, UDMA_CQ_DEFAULT_BURST_NUM); + udma_reg_write(jfc_context, CQC_CQ_PERIOD, UDMA_CQ_DEFAULT_INTERVAL); + if (udma_jfc->jfc_caps & UDMA_JFC_CAP_RECORD_DB) { + udma_reg_enable(jfc_context, CQC_DB_RECORD_EN); + udma_reg_write(jfc_context, CQC_CQE_DB_RECORD_ADDR_L, + lower_32_bits(udma_jfc->db.dma) >> + DMA_DB_RECORD_SHIFT); + udma_reg_write(jfc_context, CQC_CQE_DB_RECORD_ADDR_H, + upper_32_bits(udma_jfc->db.dma)); + } + + if (udma_jfc->jfc_attr_ex.create_flags == + UDMA_JFC_CREATE_ENABLE_POE_MODE) { + udma_reg_enable(jfc_context, CQC_POE_EN); + udma_reg_write(jfc_context, CQC_POE_NUM, + udma_jfc->jfc_attr_ex.poe_channel); + } + + if (udma_jfc->jfc_attr_ex.create_flags == UDMA_JFC_CREATE_ENABLE_NOTIFY) + set_write_notify_param(udma_dev, udma_jfc, jfc_context); +} + +static int udma_create_jfc_cqc(struct udma_dev *udma_dev, + struct udma_jfc *udma_jfc, + uint64_t *mtts, uint64_t dma_handle) +{ + struct udma_cmd_mailbox *mailbox; + int ret; + + mailbox = udma_alloc_cmd_mailbox(udma_dev); + if (IS_ERR(mailbox)) { + dev_err(udma_dev->dev, "failed to alloc mailbox for CQC.\n"); + return PTR_ERR(mailbox); + } + + udma_write_jfc_cqc(udma_dev, udma_jfc, mailbox->buf, mtts, dma_handle); + + ret = udma_hw_create_cq(udma_dev, mailbox, udma_jfc->cqn); + if (ret) + dev_err(udma_dev->dev, + "failed to send create cmd for jfc(0x%llx), ret = %d.\n", + udma_jfc->cqn, ret); + + udma_free_cmd_mailbox(udma_dev, mailbox); + + return ret; +} + +static int alloc_jfc_cqc(struct udma_dev *udma_dev, struct udma_jfc *udma_jfc) +{ + struct udma_jfc_table *jfc_table = &udma_dev->jfc_table; + uint64_t mtts[MTT_MIN_COUNT] = {}; + uint64_t dma_handle; + int ret; + + ret = udma_mtr_find(udma_dev, &udma_jfc->mtr, 0, mtts, + ARRAY_SIZE(mtts), &dma_handle); + if (!ret) { + dev_err(udma_dev->dev, + "failed to find JFC mtr, ret = %d.\n", ret); + return -EINVAL; + } + + ret = udma_table_get(udma_dev, &jfc_table->table, udma_jfc->cqn); + if (ret) { + dev_err(udma_dev->dev, + "failed to get JFC(0x%llx) context, ret = %d.\n", + udma_jfc->cqn, ret); + return ret; + } + + ret = xa_err(xa_store(&jfc_table->xa, udma_jfc->cqn, udma_jfc, + GFP_KERNEL)); + if (ret) { + dev_err(udma_dev->dev, "failed to store JFC, ret = %d.\n", ret); + goto err_put; + } + + ret = udma_create_jfc_cqc(udma_dev, udma_jfc, mtts, dma_handle); + if (ret) + goto err_xa; + + udma_jfc->arm_sn = 1; + return 0; + +err_xa: + xa_erase(&jfc_table->xa, udma_jfc->cqn); +err_put: + udma_table_put(udma_dev, &jfc_table->table, udma_jfc->cqn); + + return ret; +} + +static void store_jfc_id(struct udma_dev *udma_dev, struct udma_jfc *udma_jfc) +{ + struct jfc_list *jfc_new; + struct jfc_list *jfc_now; + unsigned long flags; + spinlock_t *lock; + int ret; + int i; + + ret = udma_find_dfx_dev(udma_dev, &i); + if (ret) + return; + + jfc_new = kzalloc(sizeof(struct jfc_list), GFP_KERNEL); + if (jfc_new == NULL) + return; + + lock = &g_udma_dfx_list[i].dfx->jfc_list->node_lock; + spin_lock_irqsave(lock, flags); + list_for_each_entry(jfc_now, + &g_udma_dfx_list[i].dfx->jfc_list->node, node) { + if (jfc_now->jfc_id == udma_jfc->cqn) + goto found; + } + + jfc_new->jfc_id = udma_jfc->cqn; + list_add(&jfc_new->node, &g_udma_dfx_list[i].dfx->jfc_list->node); + spin_unlock_irqrestore(lock, flags); + + return; + +found: + spin_unlock_irqrestore(lock, flags); + kfree(jfc_new); +} + +static void delete_jfc_id(struct udma_dev *udma_dev, struct udma_jfc *udma_jfc) +{ + struct jfc_list *jfc_now, *jfc_tmp; + unsigned long flags; + spinlock_t *lock; + int ret; + int i; + + ret = udma_find_dfx_dev(udma_dev, &i); + if (ret) + return; + + lock = &g_udma_dfx_list[i].dfx->jfc_list->node_lock; + spin_lock_irqsave(lock, flags); + list_for_each_entry_safe(jfc_now, jfc_tmp, + &g_udma_dfx_list[i].dfx->jfc_list->node, + node) { + if (jfc_now->jfc_id == udma_jfc->cqn) { + list_del(&jfc_now->node); + kfree(jfc_now); + spin_unlock_irqrestore(lock, flags); + return; + } + } + spin_unlock_irqrestore(lock, flags); +} + +static void free_jfc_cqc(struct udma_dev *udma_dev, struct udma_jfc *udma_jfc) +{ + struct udma_jfc_table *jfc_table = &udma_dev->jfc_table; + int ret; + + ret = udma_hw_destroy_cq(udma_dev, udma_jfc->cqn); + if (ret) + dev_err(udma_dev->dev, "destroy failed (%d) for JFC %06llx\n", + ret, udma_jfc->cqn); + + xa_erase(&jfc_table->xa, udma_jfc->cqn); + udma_table_put(udma_dev, &jfc_table->table, udma_jfc->cqn); +} + +static void free_jfc_buf(struct udma_dev *udma_dev, struct udma_jfc *udma_jfc) +{ + /* wait for all interrupt processed */ + if (refcount_dec_and_test(&udma_jfc->refcount)) + complete(&udma_jfc->free); + wait_for_completion(&udma_jfc->free); + + if (udma_dev->caps.flags & UDMA_CAP_FLAG_CQ_RECORD_DB) + udma_db_unmap_user(udma_dev, &udma_jfc->db); + udma_mtr_destroy(udma_dev, &udma_jfc->mtr); +} + +static uint8_t get_least_load_bankid_for_jfc(struct udma_bank *bank) +{ + uint32_t least_load = bank[0].inuse; + uint8_t bankid = 0; + uint32_t bankcnt; + uint8_t i; + + for (i = 1; i < UDMA_CQ_BANK_NUM; i++) { + bankcnt = bank[i].inuse; + if (bankcnt < least_load) { + least_load = bankcnt; + bankid = i; + } + } + + return bankid; +} + +static int alloc_jfc_id(struct udma_dev *udma_dev, struct udma_jfc *udma_jfc) +{ + struct udma_jfc_table *jfc_table = &udma_dev->jfc_table; + struct udma_bank *bank; + uint8_t bankid; + int id; + + mutex_lock(&jfc_table->bank_mutex); + bankid = get_least_load_bankid_for_jfc(jfc_table->bank); + bank = &jfc_table->bank[bankid]; + + id = ida_alloc_range(&bank->ida, bank->min, bank->max, GFP_KERNEL); + if (id < 0) { + mutex_unlock(&jfc_table->bank_mutex); + return id; + } + + /* the lower 2 bits is bankid */ + udma_jfc->cqn = (id << CQ_BANKID_SHIFT) | bankid; + bank->inuse++; + mutex_unlock(&jfc_table->bank_mutex); + udma_jfc->ubcore_jfc.id = udma_jfc->cqn; + + return 0; +} + +static void free_jfc_id(struct udma_dev *udma_dev, struct udma_jfc *udma_jfc) +{ + struct udma_jfc_table *jfc_table = &udma_dev->jfc_table; + struct udma_bank *bank; + + bank = &jfc_table->bank[get_jfc_bankid(udma_jfc->cqn)]; + ida_free(&bank->ida, udma_jfc->cqn >> CQ_BANKID_SHIFT); + + mutex_lock(&jfc_table->bank_mutex); + bank->inuse--; + mutex_unlock(&jfc_table->bank_mutex); +} + +struct ubcore_jfc *udma_create_jfc(struct ubcore_device *dev, const struct ubcore_jfc_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + struct udma_create_jfc_ucmd ucmd = {}; + struct udma_jfc *udma_jfc; + int ret; + + ret = check_create_jfc(udma_dev, cfg, &ucmd, udata); + if (ret) + goto err; + + udma_jfc = kzalloc(sizeof(*udma_jfc), GFP_KERNEL); + if (ZERO_OR_NULL_PTR(udma_jfc)) { + ret = -ENOMEM; + goto err; + } + + init_jfc(udma_jfc, &ucmd); + + set_jfc_param(udma_jfc, cfg); + + ret = alloc_jfc_buf(udma_dev, udma_jfc, udata, &ucmd); + if (ret) + goto err_jfc; + + ret = alloc_jfc_id(udma_dev, udma_jfc); + if (ret) + goto err_jfc_buf; + + ret = alloc_jfc_cqc(udma_dev, udma_jfc); + if (ret) + goto err_jfc_id; + + if (dfx_switch) + store_jfc_id(udma_dev, udma_jfc); + + return &udma_jfc->ubcore_jfc; + +err_jfc_id: + free_jfc_id(udma_dev, udma_jfc); +err_jfc_buf: + free_jfc_buf(udma_dev, udma_jfc); +err_jfc: + kfree(udma_jfc); +err: + return NULL; +} + +int udma_destroy_jfc(struct ubcore_jfc *jfc) +{ + struct udma_dev *udma_dev = to_udma_dev(jfc->ub_dev); + struct udma_jfc *udma_jfc = to_udma_jfc(jfc); + + free_jfc_cqc(udma_dev, udma_jfc); + + if (dfx_switch) + delete_jfc_id(udma_dev, udma_jfc); + + free_jfc_id(udma_dev, udma_jfc); + free_jfc_buf(udma_dev, udma_jfc); + kfree(udma_jfc); + + return 0; +} + +int udma_modify_jfc(struct ubcore_jfc *ubcore_jfc, const struct ubcore_jfc_attr *attr, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_device = to_udma_dev(ubcore_jfc->ub_dev); + uint16_t cq_period = attr->moderate_period; + uint16_t cq_count = attr->moderate_count; + struct udma_jfc_context *jfc_context; + struct udma_jfc_context *cqc_mask; + struct udma_cmd_mailbox *mailbox; + struct udma_cmq_desc desc; + struct udma_jfc *udma_jfc; + struct udma_mbox *mb; + int ret; + + udma_jfc = to_udma_jfc(ubcore_jfc); + ret = check_jfc_cfg(udma_device, &ubcore_jfc->jfc_cfg); + if (ret) + return ret; + + if (!(attr->mask & (UBCORE_JFC_MODERATE_COUNT | + UBCORE_JFC_MODERATE_PERIOD))) { + dev_err(udma_device->dev, + "JFC modify mask is not set or invalid.\n"); + return -EINVAL; + } + + mailbox = udma_alloc_cmd_mailbox(udma_device); + if (IS_ERR(mailbox)) { + dev_err(udma_device->dev, "failed to alloc mailbox for CQ.\n"); + return -ENOMEM; + } + + jfc_context = (struct udma_jfc_context *)mailbox->buf; + cqc_mask = (struct udma_jfc_context *)mailbox->buf + 1; + + memset(cqc_mask, 0xff, sizeof(*cqc_mask)); + if (attr->mask & UBCORE_JFC_MODERATE_COUNT) { + udma_reg_write(jfc_context, CQC_CQ_MAX_CNT, cq_count); + udma_reg_clear(cqc_mask, CQC_CQ_MAX_CNT); + } + + if (attr->mask & UBCORE_JFC_MODERATE_PERIOD) { + udma_reg_write(jfc_context, CQC_CQ_PERIOD, cq_period); + udma_reg_clear(cqc_mask, CQC_CQ_PERIOD); + } + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mbox_desc_init(mb, mailbox->dma, 0, ubcore_jfc->id, + UDMA_CMD_MODIFY_CQC); + + ret = udma_cmd_mbox(udma_device, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); + if (ret) + dev_err(udma_device->dev, + "failed to send modify cmd for jfc(0x%x), ret = %d.\n", + ubcore_jfc->id, ret); + udma_free_cmd_mailbox(udma_device, mailbox); + + return ret; +} + +void udma_jfc_completion(struct udma_dev *udma_dev, uint32_t cqn) +{ + struct ubcore_jfc *ubcore_jfc; + struct udma_jfc *udma_jfc; + + udma_jfc = (struct udma_jfc *)xa_load(&udma_dev->jfc_table.xa, cqn); + if (!udma_jfc) { + dev_warn(udma_dev->dev, + "Completion event for bogus CQ 0x%06x\n", cqn); + return; + } + + ubcore_jfc = &udma_jfc->ubcore_jfc; + if (ubcore_jfc->jfce_handler) + ubcore_jfc->jfce_handler(ubcore_jfc); +} + +void udma_jfc_event(struct udma_dev *udma_dev, uint32_t cqn, int event_type) +{ + struct device *dev = udma_dev->dev; + struct udma_jfc *udma_jfc; + struct ubcore_jfc *ubcore_jfc; + struct ubcore_event event; + + udma_jfc = (struct udma_jfc *)xa_load(&udma_dev->jfc_table.xa, cqn); + if (!udma_jfc) { + dev_warn(dev, "Async event for bogus CQ 0x%06x\n", cqn); + return; + } + + if (event_type != UDMA_EVENT_TYPE_JFC_ACCESS_ERROR && + event_type != UDMA_EVENT_TYPE_JFC_OVERFLOW) { + dev_err(dev, "Unexpected event type 0x%x on CQ 0x%06x\n", + event_type, cqn); + return; + } + + refcount_inc(&udma_jfc->refcount); + + ubcore_jfc = &udma_jfc->ubcore_jfc; + if (ubcore_jfc->jfae_handler) { + event.ub_dev = ubcore_jfc->ub_dev; + event.element.jfc = ubcore_jfc; + event.event_type = UBCORE_EVENT_JFC_ERR; + ubcore_jfc->jfae_handler(&event, ubcore_jfc->uctx); + } + + if (refcount_dec_and_test(&udma_jfc->refcount)) + complete(&udma_jfc->free); +} diff --git a/drivers/ub/hw/hns3/hns3_udma_jfc.h b/drivers/ub/hw/hns3/hns3_udma_jfc.h new file mode 100644 index 0000000000000000000000000000000000000000..5af78b398aea8d26cfd183c1f1f29303c4a3f16d --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_jfc.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_JFC_H +#define _UDMA_JFC_H + +#include "hns3_udma_device.h" +#include "hns3_udma_abi.h" + +struct udma_jfc { + struct ubcore_jfc ubcore_jfc; + struct udma_mtr mtr; + uint32_t jfc_caps; + uint32_t jfc_depth; + uint64_t cqn; + uint32_t arm_sn; + struct udma_db db; + spinlock_t lock; + refcount_t refcount; + struct completion free; + struct list_head sq_list; + struct list_head rq_list; + struct udma_jfc_attr_ex jfc_attr_ex; +}; + +#define UDMA_JFC_CONTEXT_SIZE 16 +struct udma_jfc_context { + uint32_t jfc_data[UDMA_JFC_CONTEXT_SIZE]; +}; + +#define UDMA_NOTIFY_MODE_4B 1UL + +enum udma_notify_device_en { + UDMA_NOTIFY_DEV, + UDMA_NOTIFY_DDR, +}; + +#define CQC_NOTIFY_ADDR_0_S 12 +#define CQC_NOTIFY_ADDR_0_M GENMASK(19, 12) +#define CQC_NOTIFY_ADDR_1_S 20 +#define CQC_NOTIFY_ADDR_1_M GENMASK(29, 20) +#define CQC_NOTIFY_ADDR_2_S 30 +#define CQC_NOTIFY_ADDR_2_M GENMASK(33, 30) +#define CQC_NOTIFY_ADDR_3_S 34 +#define CQC_NOTIFY_ADDR_3_M GENMASK(41, 34) +#define CQC_NOTIFY_ADDR_4_S 42 +#define CQC_NOTIFY_ADDR_4_M GENMASK(49, 42) +#define CQC_NOTIFY_ADDR_5_S 50 +#define CQC_NOTIFY_ADDR_5_M GENMASK(57, 50) +#define CQC_NOTIFY_ADDR_6_S 58 +#define CQC_NOTIFY_ADDR_6_M GENMASK(63, 58) + +static inline struct udma_jfc *to_udma_jfc(struct ubcore_jfc *jfc) +{ + return container_of(jfc, struct udma_jfc, ubcore_jfc); +} + +struct ubcore_jfc *udma_create_jfc(struct ubcore_device *dev, const struct ubcore_jfc_cfg *cfg, + struct ubcore_udata *udata); +int udma_destroy_jfc(struct ubcore_jfc *jfc); +int udma_modify_jfc(struct ubcore_jfc *ubcore_jfc, const struct ubcore_jfc_attr *attr, + struct ubcore_udata *udata); +void udma_jfc_completion(struct udma_dev *udma_dev, uint32_t cqn); +void udma_jfc_event(struct udma_dev *udma_dev, uint32_t cqn, int event_type); +static inline uint8_t get_jfc_bankid(uint64_t cqn) +{ + /* The lower 2 bits of CQN are used to hash to different banks */ + return (uint8_t)(cqn & GENMASK(1, 0)); +} + +#endif /* _UDMA_JFC_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_jfr.c b/drivers/ub/hw/hns3/hns3_udma_jfr.c new file mode 100644 index 0000000000000000000000000000000000000000..9deb39423bec26499264f7260fdac055ab6d190b --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_jfr.c @@ -0,0 +1,727 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include +#include "hns3_udma_hem.h" +#include "hns3_udma_cmd.h" +#include "hns3_udma_db.h" +#include "hns3_udma_jfc.h" +#include "hns3_udma_dfx.h" +#include "hns3_udma_jfr.h" + +static int init_jfr_cfg(struct udma_dev *dev, struct udma_jfr *jfr, + const struct ubcore_jfr_cfg *cfg) +{ + if (!cfg->max_sge || + cfg->depth > dev->caps.max_srq_wrs || + cfg->max_sge > dev->caps.max_srq_sges) { + dev_err(dev->dev, "invalid jfr cfg, depth = %u, sge = %u.\n", + cfg->depth, cfg->max_sge); + return -EINVAL; + } + + jfr->wqe_cnt = roundup_pow_of_two(cfg->depth); + + if (cfg->trans_mode == UBCORE_TP_UM) + /* reserved for UM header */ + jfr->max_sge = roundup_pow_of_two(cfg->max_sge + 1); + else + jfr->max_sge = roundup_pow_of_two(cfg->max_sge); + + memcpy(&jfr->ubcore_jfr.jfr_cfg, cfg, sizeof(struct ubcore_jfr_cfg)); + return 0; +} + +static int alloc_jfr_idx(struct udma_dev *dev, struct udma_jfr *jfr, + struct ubcore_udata *udata, uint64_t addr) +{ + struct udma_jfr_idx_que *idx_que = &jfr->idx_que; + struct udma_buf_attr buf_attr = {}; + int ret; + + jfr->idx_que.entry_shift = ilog2(UDMA_IDX_QUE_ENTRY_SZ); + + buf_attr.page_shift = dev->caps.idx_buf_pg_sz + PAGE_SHIFT; + buf_attr.region[0].size = + to_udma_hem_entries_size(jfr->wqe_cnt, + jfr->idx_que.entry_shift); + buf_attr.region[0].hopnum = dev->caps.idx_hop_num; + buf_attr.region_count = 1; + + ret = udma_mtr_create(dev, &idx_que->mtr, &buf_attr, + dev->caps.idx_ba_pg_sz + PAGE_SHIFT, + addr, !!udata); + if (ret) { + dev_err(dev->dev, + "failed to alloc JFR idx mtr, ret = %d.\n", ret); + return ret; + } + + idx_que->head = 0; + idx_que->tail = 0; + + return 0; +} + +static void free_jfr_idx(struct udma_dev *dev, struct udma_jfr *jfr) +{ + struct udma_jfr_idx_que *idx_que = &jfr->idx_que; + + return udma_mtr_destroy(dev, &idx_que->mtr); +} + +static int alloc_jfr_wqe_buf(struct udma_dev *dev, + struct udma_jfr *jfr, + struct ubcore_udata *udata, uint64_t addr) +{ + struct udma_buf_attr buf_attr = {}; + int ret; + + jfr->wqe_shift = ilog2(roundup_pow_of_two(UDMA_SGE_SIZE * + jfr->max_sge)); + + buf_attr.page_shift = dev->caps.srqwqe_buf_pg_sz + PAGE_SHIFT; + buf_attr.region[0].size = to_udma_hem_entries_size(jfr->wqe_cnt, + jfr->wqe_shift); + buf_attr.region[0].hopnum = dev->caps.srqwqe_hop_num; + buf_attr.region_count = 1; + + ret = udma_mtr_create(dev, &jfr->buf_mtr, &buf_attr, + dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT, + addr, !!udata); + if (ret) + dev_err(dev->dev, + "failed to alloc JFR buf mtr, ret = %d.\n", ret); + + return ret; +} + +static void free_jfr_wqe_buf(struct udma_dev *dev, struct udma_jfr *jfr) +{ + udma_mtr_destroy(dev, &jfr->buf_mtr); +} + +static int alloc_jfr_buf(struct udma_dev *dev, struct udma_jfr *jfr, + struct ubcore_udata *udata) +{ + struct udma_create_jfr_ucmd ucmd = {}; + struct udma_create_jfr_resp resp = {}; + int ret; + + if (udata) { + ret = copy_from_user(&ucmd, (void *)udata->udrv_data->in_addr, + min(udata->udrv_data->in_len, + (uint32_t)sizeof(ucmd))); + if (ret) { + dev_err(dev->dev, + "failed to copy JFR udata, ret = %d.\n", + ret); + return -EFAULT; + } + } + + ret = alloc_jfr_idx(dev, jfr, udata, ucmd.idx_addr); + if (ret) + return ret; + + ret = alloc_jfr_wqe_buf(dev, jfr, udata, ucmd.buf_addr); + if (ret) + goto err_idx; + + if (dev->caps.flags & UDMA_CAP_FLAG_SRQ_RECORD_DB) { + ret = udma_db_map_user(dev, ucmd.db_addr, &jfr->db); + if (ret) { + dev_err(dev->dev, + "map jfr db failed, ret = %d.\n", ret); + goto err_db; + } + jfr->jfr_caps |= UDMA_JFR_CAP_RECORD_DB; + } + + if (udata) { + resp.jfr_caps = jfr->jfr_caps; + ret = copy_to_user((void *)udata->udrv_data->out_addr, &resp, + min(udata->udrv_data->out_len, + (uint32_t)sizeof(resp))); + if (ret) { + dev_err(dev->dev, + "failed to copy jfr resp, ret = %d.\n", + ret); + goto err_copy; + } + } + refcount_set(&jfr->refcount, 1); + init_completion(&jfr->free); + return 0; + +err_copy: + if (dev->caps.flags & UDMA_CAP_FLAG_SRQ_RECORD_DB) { + udma_db_unmap_user(dev, &jfr->db); + jfr->jfr_caps &= ~UDMA_JFR_CAP_RECORD_DB; + } +err_db: + free_jfr_wqe_buf(dev, jfr); +err_idx: + free_jfr_idx(dev, jfr); + + return ret; +} + +static int udma_write_jfr_index_queue(struct udma_dev *dev, + struct udma_jfr *jfr, + struct udma_jfr_context *ctx) +{ +#define DMA_IDX_SHIFT 3 + struct udma_jfr_idx_que *idx_que = &jfr->idx_que; + uint64_t mtts_idx[MTT_MIN_COUNT] = {}; + uint64_t dma_handle_idx = 0; + int ret; + + /* Get physical address of idx que buf */ + ret = udma_mtr_find(dev, &idx_que->mtr, 0, mtts_idx, + ARRAY_SIZE(mtts_idx), &dma_handle_idx); + if (ret < 1) { + dev_err(dev->dev, "failed to find mtr for JFR idx, ret = %d.\n", + ret); + return -ENOBUFS; + } + + udma_reg_write(ctx, SRQC_IDX_HOP_NUM, + to_udma_hem_hopnum(dev->caps.idx_hop_num, jfr->wqe_cnt)); + + udma_reg_write(ctx, SRQC_IDX_BT_BA_L, dma_handle_idx >> DMA_IDX_SHIFT); + udma_reg_write(ctx, SRQC_IDX_BT_BA_H, + upper_32_bits(dma_handle_idx >> DMA_IDX_SHIFT)); + + udma_reg_write(ctx, SRQC_IDX_BA_PG_SZ, + to_udma_hw_page_shift(idx_que->mtr.hem_cfg.ba_pg_shift)); + udma_reg_write(ctx, SRQC_IDX_BUF_PG_SZ, + to_udma_hw_page_shift(idx_que->mtr.hem_cfg.buf_pg_shift)); + + udma_reg_write(ctx, SRQC_IDX_CUR_BLK_ADDR_L, + to_udma_hw_page_addr(mtts_idx[0])); + udma_reg_write(ctx, SRQC_IDX_CUR_BLK_ADDR_H, + upper_32_bits(to_udma_hw_page_addr(mtts_idx[0]))); + + udma_reg_write(ctx, SRQC_IDX_NXT_BLK_ADDR_L, + to_udma_hw_page_addr(mtts_idx[1])); + udma_reg_write(ctx, SRQC_IDX_NXT_BLK_ADDR_H, + upper_32_bits(to_udma_hw_page_addr(mtts_idx[1]))); + + return 0; +} + +static int write_jfrc(struct udma_dev *dev, struct udma_jfr *jfr, void *mb_buf) +{ + struct udma_jfr_context *ctx = (struct udma_jfr_context *)mb_buf; + uint64_t mtts_wqe[MTT_MIN_COUNT] = {}; + uint64_t dma_handle_wqe = 0; + int ret; + + memset(ctx, 0, sizeof(*ctx)); + + ret = udma_mtr_find(dev, &jfr->buf_mtr, 0, mtts_wqe, + ARRAY_SIZE(mtts_wqe), &dma_handle_wqe); + if (ret < 1) { + dev_err(dev->dev, "failed to find mtr for JFR WQE, ret = %d.\n", + ret); + return -ENOBUFS; + } + + udma_reg_write(ctx, SRQC_SRQ_ST, 1); + udma_reg_write(ctx, SRQC_SRQ_TYPE, 0); + udma_reg_write(ctx, SRQC_PD, 0); + udma_reg_write(ctx, SRQC_SRQN, jfr->jfrn); + udma_reg_write(ctx, SRQC_XRCD, 0); + udma_reg_write(ctx, SRQC_XRC_RSV, 0); + udma_reg_write(ctx, SRQC_SHIFT, ilog2(jfr->wqe_cnt)); + udma_reg_write(ctx, SRQC_RQWS, + jfr->max_sge <= 0 ? 0 : fls(jfr->max_sge - 1)); + + udma_reg_write(ctx, SRQC_WQE_HOP_NUM, + to_udma_hem_hopnum(dev->caps.srqwqe_hop_num, + jfr->wqe_cnt)); + + udma_reg_write(ctx, SRQC_WQE_BT_BA_L, dma_handle_wqe >> DMA_WQE_SHIFT); + udma_reg_write(ctx, SRQC_WQE_BT_BA_H, + upper_32_bits(dma_handle_wqe >> DMA_WQE_SHIFT)); + + udma_reg_write(ctx, SRQC_WQE_BA_PG_SZ, + to_udma_hw_page_shift(jfr->buf_mtr.hem_cfg.ba_pg_shift)); + udma_reg_write(ctx, SRQC_WQE_BUF_PG_SZ, + to_udma_hw_page_shift(jfr->buf_mtr.hem_cfg.buf_pg_shift)); + if (jfr->jfr_caps & UDMA_JFR_CAP_RECORD_DB) { + udma_reg_enable(ctx, SRQC_RECORD_DB_EN); + udma_reg_write(ctx, SRQC_RECORD_DB_ADDR_L, + lower_32_bits(jfr->db.dma) >> + DMA_DB_RECORD_SHIFT); + udma_reg_write(ctx, SRQC_RECORD_DB_ADDR_H, + upper_32_bits(jfr->db.dma)); + } + + return udma_write_jfr_index_queue(dev, jfr, ctx); +} + +static int udma_hw_create_srq(struct udma_dev *dev, + struct udma_cmd_mailbox *mailbox, + uint64_t jfrn) +{ + struct udma_cmq_desc desc; + struct udma_mbox *mb; + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mbox_desc_init(mb, mailbox->dma, 0, jfrn, UDMA_CMD_CREATE_SRQ); + + return udma_cmd_mbox(dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); +} + +static int alloc_jfrc(struct udma_dev *dev, struct udma_jfr *jfr) +{ + struct udma_jfr_table *jfr_table = &dev->jfr_table; + struct udma_ida *jfr_ida = &jfr_table->jfr_ida; + struct udma_cmd_mailbox *mailbox; + int ret; + int id; + + id = ida_alloc_range(&jfr_ida->ida, jfr_ida->min, jfr_ida->max, + GFP_KERNEL); + if (id < 0) { + dev_err(dev->dev, "failed to alloc jfr_id(%d).\n", id); + return id; + } + jfr->jfrn = (uint32_t)id; + jfr->ubcore_jfr.id = (uint32_t)id; + + ret = udma_table_get(dev, &jfr_table->table, jfr->jfrn); + if (ret) { + dev_err(dev->dev, "failed to get JFRC table, ret = %d.\n", ret); + goto err_ida; + } + + ret = xa_err(xa_store(&jfr_table->xa, jfr->jfrn, jfr, GFP_KERNEL)); + if (ret) { + dev_err(dev->dev, "failed to store JFRC, ret = %d.\n", ret); + goto err_put; + } + + mailbox = udma_alloc_cmd_mailbox(dev); + if (IS_ERR_OR_NULL(mailbox)) { + dev_err(dev->dev, "failed to alloc mailbox for JFRC.\n"); + ret = -ENOMEM; + goto err_xa; + } + + ret = write_jfrc(dev, jfr, mailbox->buf); + if (ret) { + dev_err(dev->dev, "failed to write JFRC.\n"); + goto err_mbox; + } + + ret = udma_hw_create_srq(dev, mailbox, jfr->jfrn); + if (ret) { + dev_err(dev->dev, "failed to config JFRC, ret = %d.\n", ret); + goto err_mbox; + } + + udma_free_cmd_mailbox(dev, mailbox); + + return 0; + +err_mbox: + udma_free_cmd_mailbox(dev, mailbox); +err_xa: + xa_erase(&jfr_table->xa, jfr->jfrn); +err_put: + udma_table_put(dev, &jfr_table->table, jfr->jfrn); +err_ida: + ida_free(&jfr_ida->ida, id); + + return ret; +} + +static int udma_hw_destroy_srq(struct udma_dev *dev, uint64_t jfrn) +{ + struct udma_cmq_desc desc; + struct udma_mbox *mb; + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mbox_desc_init(mb, 0, 0, jfrn, UDMA_CMD_DESTROY_SRQ); + + return udma_cmd_mbox(dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); +} + +static void store_jfr_id(struct udma_dev *dev, struct udma_jfr *jfr) +{ + struct jfr_list *jfr_new; + struct jfr_list *jfr_now; + unsigned long flags; + spinlock_t *lock; + int ret; + int i; + + ret = udma_find_dfx_dev(dev, &i); + if (ret) + return; + + jfr_new = kzalloc(sizeof(struct jfr_list), GFP_KERNEL); + if (jfr_new == NULL) + return; + + lock = &g_udma_dfx_list[i].dfx->jfr_list->node_lock; + spin_lock_irqsave(lock, flags); + list_for_each_entry(jfr_now, + &g_udma_dfx_list[i].dfx->jfr_list->node, + node) { + if (jfr_now->jfr_id == jfr->jfrn) { + jfr_now->jfc_id = jfr->ubcore_jfr.jfr_cfg.jfc->id; + goto found; + } + } + + jfr_new->jfr_id = jfr->jfrn; + jfr_new->jfc_id = jfr->ubcore_jfr.jfr_cfg.jfc->id; + list_add(&jfr_new->node, &g_udma_dfx_list[i].dfx->jfr_list->node); + spin_unlock_irqrestore(lock, flags); + + return; + +found: + spin_unlock_irqrestore(lock, flags); + kfree(jfr_new); +} + +static void delete_jfr_id(struct udma_dev *dev, struct udma_jfr *jfr) +{ + struct jfr_list *jfr_now, *jfr_tmp; + unsigned long flags; + spinlock_t *lock; + int ret; + int i; + + ret = udma_find_dfx_dev(dev, &i); + if (ret) + return; + + lock = &g_udma_dfx_list[i].dfx->jfr_list->node_lock; + spin_lock_irqsave(lock, flags); + list_for_each_entry_safe(jfr_now, jfr_tmp, + &g_udma_dfx_list[i].dfx->jfr_list->node, + node) { + if (jfr_now->jfr_id == jfr->jfrn) { + list_del(&jfr_now->node); + kfree(jfr_now); + spin_unlock_irqrestore(lock, flags); + return; + } + } + spin_unlock_irqrestore(lock, flags); +} + +static void free_jfrc(struct udma_dev *dev, uint32_t jfrn) +{ + struct udma_jfr_table *jfr_table = &dev->jfr_table; + int ret; + + ret = udma_hw_destroy_srq(dev, jfrn); + if (ret) + dev_err(dev->dev, "destroy failed (%d) for JFRN 0x%06x\n", + ret, jfrn); + + xa_erase(&jfr_table->xa, jfrn); + + udma_table_put(dev, &jfr_table->table, jfrn); + + ida_free(&jfr_table->jfr_ida.ida, (int)jfrn); +} + +static void free_jfr_buf(struct udma_dev *dev, struct udma_jfr *jfr) +{ + if (refcount_dec_and_test(&jfr->refcount)) + complete(&jfr->free); + wait_for_completion(&jfr->free); + + if (dev->caps.flags & UDMA_CAP_FLAG_SRQ_RECORD_DB) + udma_db_unmap_user(dev, &jfr->db); + free_jfr_wqe_buf(dev, jfr); + free_jfr_idx(dev, jfr); +} + +static int udma_modify_jfr_um_qpc(struct udma_dev *dev, struct udma_jfr *jfr, + enum udma_qp_state target_state) +{ + union ubcore_tp_attr_mask ubcore_attr_mask; + struct udma_modify_tp_attr attr = {}; + struct udma_qp *qp = jfr->um_qp; + int ret; + + attr.path_mtu = UBCORE_MTU_4096; + qp->udma_device = dev; + qp->qp_attr.jfr = jfr; + qp->recv_jfc = to_udma_jfc(jfr->ubcore_jfr.jfr_cfg.jfc); + qp->send_jfc = NULL; + ubcore_attr_mask.value = 0; + qp->m_attr = &attr; + + ret = udma_modify_qp_common(qp, NULL, ubcore_attr_mask, jfr->um_qp->state, target_state); + if (ret) + dev_err(dev->dev, "failed to modify qpc to RTR.\n"); + + qp->state = target_state; + return ret; +} + +static int alloc_jfr_um_qp(struct udma_dev *dev, struct udma_jfr *jfr) +{ + struct ubcore_udata udata; + struct udma_qp *qp; + int ret; + + qp = kzalloc(sizeof(*qp), GFP_KERNEL); + if (!qp) + return -ENOMEM; + + qp->qp_type = QPT_UD; + qp->qp_attr.qp_type = QPT_UD; + qp->qp_attr.qpn_map = &jfr->qpn_map; + qp->qp_attr.recv_jfc = to_udma_jfc(jfr->ubcore_jfr.jfr_cfg.jfc); + qp->qp_attr.send_jfc = NULL; + udata.uctx = NULL; + ret = udma_create_qp_common(dev, qp, &udata); + if (ret) { + dev_err(dev->dev, "failed to create qpc.\n"); + goto failed_create_qpc; + } + jfr->um_qp = qp; + + qp->state = QPS_RESET; + ret = udma_modify_jfr_um_qpc(dev, jfr, QPS_RTR); + if (ret) { + dev_err(dev->dev, "failed to modify qpc.\n"); + goto failed_modify_qpc; + } + + return 0; + +failed_modify_qpc: + udma_destroy_qp_common(dev, qp); +failed_create_qpc: + kfree(qp); + + return ret; +} + +void destroy_jfr_um_qp(struct udma_dev *dev, struct udma_jfr *jfr) +{ + udma_modify_jfr_um_qpc(dev, jfr, QPS_RESET); + udma_destroy_qp_common(dev, jfr->um_qp); + kfree(jfr->um_qp); +} + +struct ubcore_jfr *udma_create_jfr(struct ubcore_device *dev, const struct ubcore_jfr_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + struct udma_jfr *jfr; + int ret; + + jfr = kcalloc(1, sizeof(*jfr), GFP_KERNEL); + if (!jfr) + return NULL; + + ret = init_jfr_cfg(udma_dev, jfr, cfg); + if (ret) + goto err_alloc_jfr; + + ret = alloc_jfr_buf(udma_dev, jfr, udata); + if (ret) + goto err_alloc_jfr; + + ret = alloc_jfrc(udma_dev, jfr); + if (ret) + goto err_alloc_buf; + + xa_init(&jfr->tp_table_xa); + init_jetty_x_qpn_bitmap(udma_dev, &jfr->qpn_map, + udma_dev->caps.num_jfr_shift, + UDMA_JFR_QPN_PREFIX, jfr->jfrn); + if (cfg->trans_mode == UBCORE_TP_UM) { + ret = alloc_jfr_um_qp(udma_dev, jfr); + if (ret) + goto err_alloc_jfrc; + } + + if (dfx_switch) + store_jfr_id(udma_dev, jfr); + + return &jfr->ubcore_jfr; + +err_alloc_jfrc: + clean_jetty_x_qpn_bitmap(&jfr->qpn_map); + free_jfrc(udma_dev, jfr->jfrn); +err_alloc_buf: + free_jfr_buf(udma_dev, jfr); +err_alloc_jfr: + kfree(jfr); + + return NULL; +} + +int udma_destroy_jfr(struct ubcore_jfr *jfr) +{ + struct udma_dev *dev = to_udma_dev(jfr->ub_dev); + struct udma_jfr *udma_jfr = to_udma_jfr(jfr); + + if (udma_jfr->um_qp) + destroy_jfr_um_qp(dev, udma_jfr); + clean_jetty_x_qpn_bitmap(&udma_jfr->qpn_map); + + if (dfx_switch) + delete_jfr_id(dev, udma_jfr); + + free_jfrc(dev, udma_jfr->jfrn); + free_jfr_buf(dev, udma_jfr); + kfree(jfr); + + return 0; +} + +struct udma_jfr *get_udma_jfr(struct ubcore_device *dev, uint32_t jfr_id) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + struct udma_jfr *udma_jfr; + + udma_jfr = (struct udma_jfr *)xa_load(&udma_dev->jfr_table.xa, jfr_id); + if (IS_ERR_OR_NULL(udma_jfr)) { + dev_err(&dev->dev, "failed to find jfr, jfr_id:%u.", jfr_id); + return NULL; + } + + return udma_jfr; +} + +struct ubcore_tjetty *udma_import_jfr(struct ubcore_device *dev, + const struct ubcore_tjetty_cfg *cfg, + struct ubcore_udata *udata) +{ + struct ubcore_tjetty *tjfr; + + tjfr = kcalloc(1, sizeof(*tjfr), GFP_KERNEL); + if (!tjfr) + return NULL; + + return tjfr; +} + +int udma_unimport_jfr(struct ubcore_tjetty *tjfr) +{ + kfree(tjfr); + + return 0; +} + +static int udma_hw_modify_srq(struct udma_dev *dev, uint32_t jfrn, + uint16_t jfr_limit) +{ + struct udma_jfr_context *jfr_context; + struct udma_jfr_context *jfrc_mask; + struct udma_cmd_mailbox *mailbox; + struct udma_cmq_desc desc; + struct udma_mbox *mb; + int ret; + + mailbox = udma_alloc_cmd_mailbox(dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + jfr_context = (struct udma_jfr_context *)mailbox->buf; + jfrc_mask = (struct udma_jfr_context *)mailbox->buf + 1; + memset(jfrc_mask, 0xff, sizeof(*jfrc_mask)); + udma_reg_write(jfr_context, SRQC_LIMIT_WL, jfr_limit); + udma_reg_clear(jfrc_mask, SRQC_LIMIT_WL); + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mbox_desc_init(mb, mailbox->dma, 0, jfrn, UDMA_CMD_MODIFY_SRQC); + + ret = udma_cmd_mbox(dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); + if (ret) + dev_err(dev->dev, "modify JFR(%u) cmd error(%d).\n", + jfrn, ret); + udma_free_cmd_mailbox(dev, mailbox); + + return ret; +} + +int udma_modify_jfr(struct ubcore_jfr *jfr, const struct ubcore_jfr_attr *attr, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_dev = to_udma_dev(jfr->ub_dev); + struct udma_jfr *udma_jfr = to_udma_jfr(jfr); + uint32_t jfr_limit; + int ret; + + if (!(attr->mask & UBCORE_JFR_RX_THRESHOLD)) { + dev_err(udma_dev->dev, "JFR threshold mask is not set.\n"); + return -EINVAL; + } + + jfr_limit = attr->rx_threshold; + if (jfr_limit > udma_jfr->wqe_cnt) { + dev_err(udma_dev->dev, + "JFR limit(%u) larger than wqe num(%u).\n", + jfr_limit, udma_jfr->wqe_cnt); + return -EINVAL; + } + + ret = udma_hw_modify_srq(udma_dev, udma_jfr->jfrn, jfr_limit); + if (ret) + dev_err(udma_dev->dev, + "hw modify srq failed, ret = %d.\n", ret); + + return ret; +} + +void udma_jfr_event(struct udma_dev *udma_dev, uint32_t jfrn, int event_type) +{ + struct udma_jfr_table *jfr_table = &udma_dev->jfr_table; + struct ubcore_jfr *ubcore_jfr; + struct udma_jfr *jfr; + struct ubcore_event event; + + xa_lock(&jfr_table->xa); + jfr = (struct udma_jfr *)xa_load(&jfr_table->xa, jfrn); + xa_unlock(&jfr_table->xa); + + if (!jfr) { + dev_warn(udma_dev->dev, "Async event for bogus SRQ 0x%08x\n", + jfrn); + return; + } + + event.event_type = UBCORE_EVENT_JFR_ACCESS_ERR; + + refcount_inc(&jfr->refcount); + ubcore_jfr = &jfr->ubcore_jfr; + if (ubcore_jfr->jfae_handler) { + event.ub_dev = ubcore_jfr->ub_dev; + event.element.jfr = ubcore_jfr; + ubcore_jfr->jfae_handler(&event, ubcore_jfr->uctx); + dev_info(udma_dev->dev, "Async event for JFR 0x%08x\n", jfrn); + } + + if (refcount_dec_and_test(&jfr->refcount)) + complete(&jfr->free); +} diff --git a/drivers/ub/hw/hns3/hns3_udma_jfr.h b/drivers/ub/hw/hns3/hns3_udma_jfr.h new file mode 100644 index 0000000000000000000000000000000000000000..c804889e3ac14964f9d66babf692e299eff03617 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_jfr.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_JFR_H +#define _UDMA_JFR_H + +#include "hns3_udma_common.h" +#include "hns3_udma_qp.h" + +struct udma_jfr_idx_que { + struct udma_mtr mtr; + int entry_shift; + uint32_t head; + uint32_t tail; +}; + +struct udma_jfr { + struct ubcore_jfr ubcore_jfr; + uint32_t jfrn; + uint32_t wqe_cnt; + uint32_t max_sge; + uint32_t wqe_shift; + struct udma_jfr_idx_que idx_que; + struct udma_mtr buf_mtr; + struct udma_db db; + struct udma_qp *um_qp; + struct xarray tp_table_xa; + + refcount_t refcount; + struct completion free; + + struct udma_qpn_bitmap qpn_map; + void (*event)(struct udma_jfr *jfr, enum udma_event event_type); + uint32_t jfr_caps; +}; + +struct udma_jfr_context { + uint32_t data[16]; +}; + +#define SRQC_FIELD_LOC(h, l) ((uint64_t)(h) << 32 | (l)) + +#define SRQC_SRQ_ST SRQC_FIELD_LOC(1, 0) +#define SRQC_WQE_HOP_NUM SRQC_FIELD_LOC(3, 2) +#define SRQC_SHIFT SRQC_FIELD_LOC(7, 4) +#define SRQC_SRQN SRQC_FIELD_LOC(31, 8) +#define SRQC_LIMIT_WL SRQC_FIELD_LOC(47, 32) +#define SRQC_XRCD SRQC_FIELD_LOC(87, 64) +#define SRQC_WQE_BT_BA_L SRQC_FIELD_LOC(159, 128) +#define SRQC_WQE_BT_BA_H SRQC_FIELD_LOC(188, 160) +#define SRQC_SRQ_TYPE SRQC_FIELD_LOC(191, 191) +#define SRQC_PD SRQC_FIELD_LOC(215, 192) +#define SRQC_RQWS SRQC_FIELD_LOC(219, 216) +#define SRQC_IDX_BT_BA_L SRQC_FIELD_LOC(255, 224) +#define SRQC_IDX_BT_BA_H SRQC_FIELD_LOC(284, 256) +#define SRQC_IDX_CUR_BLK_ADDR_L SRQC_FIELD_LOC(319, 288) +#define SRQC_IDX_CUR_BLK_ADDR_H SRQC_FIELD_LOC(339, 320) +#define SRQC_IDX_HOP_NUM SRQC_FIELD_LOC(343, 342) +#define SRQC_IDX_BA_PG_SZ SRQC_FIELD_LOC(347, 344) +#define SRQC_IDX_BUF_PG_SZ SRQC_FIELD_LOC(351, 348) +#define SRQC_IDX_NXT_BLK_ADDR_L SRQC_FIELD_LOC(383, 352) +#define SRQC_IDX_NXT_BLK_ADDR_H SRQC_FIELD_LOC(403, 384) +#define SRQC_XRC_RSV SRQC_FIELD_LOC(439, 416) +#define SRQC_WQE_BA_PG_SZ SRQC_FIELD_LOC(443, 440) +#define SRQC_WQE_BUF_PG_SZ SRQC_FIELD_LOC(447, 444) +#define SRQC_RECORD_DB_EN SRQC_FIELD_LOC(448, 448) +#define SRQC_RECORD_DB_ADDR_L SRQC_FIELD_LOC(479, 449) +#define SRQC_RECORD_DB_ADDR_H SRQC_FIELD_LOC(511, 480) + +static inline struct udma_jfr *to_udma_jfr(struct ubcore_jfr *ubcore_jfr) +{ + return container_of(ubcore_jfr, struct udma_jfr, ubcore_jfr); +} + +struct ubcore_jfr *udma_create_jfr(struct ubcore_device *dev, const struct ubcore_jfr_cfg *cfg, + struct ubcore_udata *udata); +int udma_destroy_jfr(struct ubcore_jfr *jfr); +struct udma_jfr *get_udma_jfr(struct ubcore_device *dev, uint32_t jfr_id); +struct ubcore_tjetty *udma_import_jfr(struct ubcore_device *dev, + const struct ubcore_tjetty_cfg *cfg, + struct ubcore_udata *udata); +int udma_unimport_jfr(struct ubcore_tjetty *tjfr); +int udma_modify_jfr(struct ubcore_jfr *jfr, const struct ubcore_jfr_attr *attr, + struct ubcore_udata *udata); +void udma_jfr_event(struct udma_dev *udma_dev, uint32_t jfrn, int event_type); + +#endif /* _UDMA_JFR_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_jfs.c b/drivers/ub/hw/hns3/hns3_udma_jfs.c new file mode 100644 index 0000000000000000000000000000000000000000..9e4944c5d7327042abc52c927750343be4c8e023 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_jfs.c @@ -0,0 +1,333 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include "urma/ubcore_uapi.h" +#include "hns3_udma_abi.h" +#include "hns3_udma_hem.h" +#include "hns3_udma_tp.h" +#include "hns3_udma_jfc.h" +#include "hns3_udma_dfx.h" +#include "hns3_udma_jfs.h" + +static int init_jfs_cfg(struct udma_dev *dev, struct udma_jfs *jfs, + const struct ubcore_jfs_cfg *cfg) +{ + if (!cfg->max_sge || + cfg->depth > dev->caps.max_wqes || + cfg->max_sge > dev->caps.max_sq_sg) { + dev_err(dev->dev, "invalid jfs cfg, depth = %u, sge = %u.\n", + cfg->depth, cfg->max_sge); + return -EINVAL; + } + memcpy(&jfs->ubcore_jfs.jfs_cfg, cfg, sizeof(struct ubcore_jfs_cfg)); + jfs->tp_mode = cfg->trans_mode; + + return 0; +} + +static int udma_modify_jfs_um_qp(struct udma_dev *dev, struct udma_jfs *jfs, + enum udma_qp_state target_state) +{ + union ubcore_tp_attr_mask ubcore_attr_mask; + struct udma_modify_tp_attr m_attr = {}; + struct udma_qp *qp; + int ret; + + qp = &jfs->um_qp; + qp->udma_device = dev; + qp->send_jfc = qp->qp_attr.send_jfc; + qp->recv_jfc = qp->qp_attr.recv_jfc; + qp->ubcore_path_mtu = UBCORE_MTU_4096; + qp->path_mtu = UDMA_MTU_4096; + + m_attr.path_mtu = UBCORE_MTU_4096; + m_attr.hop_limit = MAX_HOP_LIMIT; + ubcore_attr_mask.value = 0; + qp->m_attr = &m_attr; + + ret = udma_modify_qp_common(qp, NULL, ubcore_attr_mask, qp->state, target_state); + if (ret) + dev_err(dev->dev, "failed to modify qpc to RTS.\n"); + + jfs->um_qp.state = target_state; + + return ret; +} + +static void udma_fill_jfs_um_qp_attr(struct udma_dev *dev, struct udma_jfs *jfs, + struct udma_qp_attr *qp_attr, + struct ubcore_ucontext *uctx, + const struct ubcore_jfs_cfg *cfg) +{ + struct udma_ucontext *udma_ctx = to_udma_ucontext(uctx); + + qp_attr->is_jetty = false; + qp_attr->jfs = jfs; + qp_attr->uctx = uctx; + qp_attr->pdn = udma_ctx->pdn; + qp_attr->cap.max_send_wr = cfg->depth; + qp_attr->cap.max_send_sge = cfg->max_sge; + qp_attr->cap.max_inline_data = cfg->max_inline_data; + qp_attr->cap.retry_cnt = cfg->retry_cnt; + qp_attr->cap.rnr_retry = cfg->rnr_retry; + qp_attr->cap.ack_timeout = cfg->err_timeout; + qp_attr->qp_type = QPT_UD; + qp_attr->recv_jfc = NULL; + qp_attr->send_jfc = to_udma_jfc(cfg->jfc); + if (jfs->ubcore_jfs.jfs_cfg.priority >= dev->caps.sl_num) { + qp_attr->priority = dev->caps.sl_num > 0 ? + dev->caps.sl_num - 1 : 0; + dev_err(dev->dev, + "The setted priority (%d) should smaller than the max priority (%d), priority (%d) is used\n", + jfs->ubcore_jfs.jfs_cfg.priority, dev->caps.sl_num, + qp_attr->priority); + } else { + qp_attr->priority = jfs->ubcore_jfs.jfs_cfg.priority; + } +} + +static int create_jfs_um_qp(struct udma_dev *dev, struct udma_jfs *jfs, + const struct ubcore_jfs_cfg *cfg, struct ubcore_udata *udata) +{ + int ret; + + udma_fill_jfs_um_qp_attr(dev, jfs, &jfs->um_qp.qp_attr, udata->uctx, cfg); + jfs->um_qp.qp_attr.qpn_map = &jfs->qpn_map; + ret = udma_create_qp_common(dev, &jfs->um_qp, udata); + if (ret) + dev_err(dev->dev, "failed to create qp for um jfs.\n"); + + return ret; +} + +int destroy_jfs_qp(struct udma_dev *dev, struct udma_jfs *jfs) +{ + int ret = 0; + + if (jfs->tp_mode == UBCORE_TP_UM) { + ret = udma_modify_jfs_um_qp(dev, jfs, QPS_RESET); + if (ret) + dev_err(dev->dev, + "failed to modify qp(0x%llx) to RESET for um jfs.\n", + jfs->um_qp.qpn); + + udma_destroy_qp_common(dev, &jfs->um_qp); + } + + return ret; +} + +static int alloc_jfs_buf(struct udma_dev *udma_dev, struct udma_jfs *jfs, + const struct ubcore_jfs_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_create_jfs_ucmd ucmd = {}; + int ret = 0; + + if (udata) { + ret = copy_from_user(&ucmd, (void *)udata->udrv_data->in_addr, + min(udata->udrv_data->in_len, + (uint32_t)sizeof(ucmd))); + if (ret) { + dev_err(udma_dev->dev, + "failed to copy jfs udata, ret = %d.\n", ret); + return -EFAULT; + } + } + + if (cfg->trans_mode == UBCORE_TP_RM) { + xa_init(&jfs->node_table); + } else if (cfg->trans_mode == UBCORE_TP_UM) { + ret = create_jfs_um_qp(udma_dev, jfs, cfg, udata); + if (ret) + return ret; + + jfs->um_qp.state = QPS_RESET; + ret = udma_modify_jfs_um_qp(udma_dev, jfs, QPS_RTS); + if (ret) + udma_destroy_qp_common(udma_dev, &jfs->um_qp); + } + + return ret; +} + +static int alloc_jfs_id(struct udma_dev *udma_dev, struct udma_jfs *jfs) +{ + struct udma_jfs_table *jfs_table = &udma_dev->jfs_table; + struct udma_ida *jfs_ida = &jfs_table->jfs_ida; + int ret; + int id; + + id = ida_alloc_range(&jfs_ida->ida, jfs_ida->min, jfs_ida->max, + GFP_KERNEL); + if (id < 0) { + dev_err(udma_dev->dev, "failed to alloc jfs_id(%d).\n", id); + return id; + } + jfs->jfs_id = (uint32_t)id; + jfs->ubcore_jfs.id = jfs->jfs_id; + + ret = xa_err(xa_store(&jfs_table->xa, jfs->jfs_id, jfs, GFP_KERNEL)); + if (ret) { + dev_err(udma_dev->dev, "failed to store JFS, ret = %d.\n", ret); + ida_free(&jfs_ida->ida, id); + } + + return ret; +} + +static void store_jfs_id(struct udma_dev *udma_dev, struct udma_jfs *jfs) +{ + struct jfs_list *jfs_new; + struct jfs_list *jfs_now; + unsigned long flags; + spinlock_t *lock; + int ret; + int i; + + ret = udma_find_dfx_dev(udma_dev, &i); + if (ret) + return; + + jfs_new = kzalloc(sizeof(struct jfs_list), GFP_KERNEL); + if (jfs_new == NULL) + return; + + lock = &g_udma_dfx_list[i].dfx->jfs_list->node_lock; + spin_lock_irqsave(lock, flags); + list_for_each_entry(jfs_now, + &g_udma_dfx_list[i].dfx->jfs_list->node, node) { + if (jfs_now->jfs_id == jfs->jfs_id) { + jfs_now->depth = jfs->ubcore_jfs.jfs_cfg.depth; + jfs_now->pri = jfs->ubcore_jfs.jfs_cfg.priority; + jfs_now->jfc_id = jfs->ubcore_jfs.jfs_cfg.jfc->id; + goto found; + } + } + + jfs_new->jfs_id = jfs->jfs_id; + jfs_new->depth = jfs->ubcore_jfs.jfs_cfg.depth; + jfs_new->pri = jfs->ubcore_jfs.jfs_cfg.priority; + jfs_new->jfc_id = jfs->ubcore_jfs.jfs_cfg.jfc->id; + list_add(&jfs_new->node, &g_udma_dfx_list[i].dfx->jfs_list->node); + spin_unlock_irqrestore(lock, flags); + + return; + +found: + spin_unlock_irqrestore(lock, flags); + kfree(jfs_new); +} + +static void delete_jfs_id(struct udma_dev *udma_dev, struct udma_jfs *jfs) +{ + struct jfs_list *jfs_now, *jfs_tmp; + unsigned long flags; + spinlock_t *lock; + int ret; + int i; + + ret = udma_find_dfx_dev(udma_dev, &i); + if (ret) + return; + + lock = &g_udma_dfx_list[i].dfx->jfs_list->node_lock; + spin_lock_irqsave(lock, flags); + list_for_each_entry_safe(jfs_now, jfs_tmp, + &g_udma_dfx_list[i].dfx->jfs_list->node, + node) { + if (jfs_now->jfs_id == jfs->jfs_id) { + list_del(&jfs_now->node); + kfree(jfs_now); + spin_unlock_irqrestore(lock, flags); + return; + } + } + spin_unlock_irqrestore(lock, flags); +} + +static void free_jfs_id(struct udma_dev *udma_dev, struct udma_jfs *jfs) +{ + struct udma_jfs_table *jfs_table = &udma_dev->jfs_table; + struct udma_ida *jfs_ida = &jfs_table->jfs_ida; + + xa_erase(&jfs_table->xa, jfs->jfs_id); + ida_free(&jfs_ida->ida, (int)jfs->jfs_id); +} + +struct ubcore_jfs *udma_create_jfs(struct ubcore_device *dev, const struct ubcore_jfs_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + struct udma_jfs *jfs; + int ret; + + jfs = kcalloc(1, sizeof(struct udma_jfs), GFP_KERNEL); + if (!jfs) + return NULL; + + ret = init_jfs_cfg(udma_dev, jfs, cfg); + if (ret) + goto err_init_cfg; + + ret = alloc_jfs_id(udma_dev, jfs); + if (ret) + goto err_alloc_jfs_id; + + init_jetty_x_qpn_bitmap(udma_dev, &jfs->qpn_map, + udma_dev->caps.num_jfs_shift, + UDMA_JFS_QPN_PREFIX, jfs->jfs_id); + + ret = alloc_jfs_buf(udma_dev, jfs, cfg, udata); + if (ret) { + dev_err(udma_dev->dev, "alloc jfs buf failed.\n"); + goto err_alloc_jfs_buf; + } + + if (dfx_switch) + store_jfs_id(udma_dev, jfs); + + return &jfs->ubcore_jfs; + +err_alloc_jfs_buf: + clean_jetty_x_qpn_bitmap(&jfs->qpn_map); + free_jfs_id(udma_dev, jfs); +err_alloc_jfs_id: +err_init_cfg: + kfree(jfs); + + return NULL; +} + +int udma_destroy_jfs(struct ubcore_jfs *jfs) +{ + struct udma_jfs *udma_jfs; + struct udma_dev *udma_dev; + int ret; + + udma_jfs = to_udma_jfs(jfs); + udma_dev = to_udma_dev(jfs->ub_dev); + + ret = destroy_jfs_qp(udma_dev, udma_jfs); + clean_jetty_x_qpn_bitmap(&udma_jfs->qpn_map); + + if (dfx_switch) + delete_jfs_id(udma_dev, udma_jfs); + + free_jfs_id(udma_dev, udma_jfs); + kfree(udma_jfs); + + return ret; +} diff --git a/drivers/ub/hw/hns3/hns3_udma_jfs.h b/drivers/ub/hw/hns3/hns3_udma_jfs.h new file mode 100644 index 0000000000000000000000000000000000000000..eee4058bb0b3ee919467cdcdef4a9ebf14f67641 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_jfs.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_JFS_H +#define _UDMA_JFS_H + +#include "hns3_udma_qp.h" +struct udma_jfs { + struct ubcore_jfs ubcore_jfs; + uint32_t jfs_id; + enum ubcore_transport_mode tp_mode; + union { + struct xarray node_table; + struct udma_qp um_qp; + }; + struct udma_qpn_bitmap qpn_map; +}; + +static inline struct udma_jfs *to_udma_jfs(struct ubcore_jfs *jfs) +{ + return container_of(jfs, struct udma_jfs, ubcore_jfs); +} + +struct ubcore_jfs *udma_create_jfs(struct ubcore_device *dev, + const struct ubcore_jfs_cfg *cfg, + struct ubcore_udata *udata); +int udma_destroy_jfs(struct ubcore_jfs *ubcore_jfs); + +#endif /* _UDMA_JFS_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_main.c b/drivers/ub/hw/hns3/hns3_udma_main.c new file mode 100644 index 0000000000000000000000000000000000000000..ab105f589a6461a3b9b9b3461a61e3e20960dc3f --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_main.c @@ -0,0 +1,1499 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include +#include +#include "urma/ubcore_api.h" +#include "hns3_udma_abi.h" +#include "hnae3.h" +#include "hns3_udma_device.h" +#include "hns3_udma_hem.h" +#include "hns3_udma_tp.h" +#include "hns3_udma_jfr.h" +#include "hns3_udma_jfc.h" +#include "hns3_udma_jfs.h" +#include "hns3_udma_segment.h" +#include "hns3_udma_jetty.h" +#include "hns3_udma_dca.h" +#include "hns3_udma_cmd.h" +#include "hns3_udma_dfx.h" + +static int udma_set_eid(struct ubcore_device *dev, union ubcore_eid eid) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + uint8_t port = 0; + + if (port >= udma_dev->caps.num_ports) + return -EINVAL; + + return udma_dev->hw->set_eid(udma_dev, eid); +} + +static int udma_uar_alloc(struct udma_dev *udma_dev, struct udma_uar *uar) +{ + struct udma_ida *uar_ida = &udma_dev->uar_ida; + int id; + + /* Using bitmap to manager UAR index */ + id = ida_alloc_range(&uar_ida->ida, uar_ida->min, uar_ida->max, + GFP_KERNEL); + if (id < 0) { + dev_err(udma_dev->dev, "failed to alloc uar id(%d).\n", id); + return id; + } + uar->logic_idx = (uint64_t)id; + + if (uar->logic_idx > 0 && udma_dev->caps.phy_num_uars > 1) + uar->index = (uar->logic_idx - 1) % + (udma_dev->caps.phy_num_uars - 1) + 1; + else + uar->index = 0; + + uar->pfn = ((pci_resource_start(udma_dev->pci_dev, + UDMA_DEV_START_OFFSET)) >> PAGE_SHIFT); + if (udma_dev->caps.flags & UDMA_CAP_FLAG_DIRECT_WQE) + udma_dev->dwqe_page = + pci_resource_start(udma_dev->pci_dev, + UDMA_DEV_EX_START_OFFSET); + + return 0; +} + +static int udma_init_ctx_resp(struct udma_dev *dev, struct ubcore_udrv_priv *udrv_data, + struct udma_dca_ctx *dca_ctx) +{ + struct udma_create_ctx_resp resp = {}; + int ret; + + resp.num_comp_vectors = dev->caps.num_comp_vectors; + resp.num_qps_shift = dev->caps.num_qps_shift; + resp.num_jfs_shift = dev->caps.num_jfs_shift; + resp.num_jfr_shift = dev->caps.num_jfr_shift; + resp.num_jetty_shift = dev->caps.num_jetty_shift; + resp.max_jfc_cqe = dev->caps.max_cqes; + resp.cqe_size = dev->caps.cqe_sz; + resp.max_jfr_wr = dev->caps.max_srq_wrs; + resp.max_jfr_sge = dev->caps.max_srq_sges; + resp.max_jfs_wr = dev->caps.max_wqes; + resp.max_jfs_sge = dev->caps.max_sq_sg; + resp.poe_ch_num = dev->caps.poe_ch_num; + resp.db_addr = pci_resource_start(dev->pci_dev, UDMA_DEV_START_OFFSET) + + UDMA_DB_ADDR_OFFSET; + + if (dev->caps.flags & UDMA_CAP_FLAG_DCA_MODE) { + resp.dca_qps = dca_ctx->max_qps; + resp.dca_mmap_size = dca_ctx->status_npage * PAGE_SIZE; + resp.dca_mode = dev->caps.flags & UDMA_CAP_FLAG_DCA_MODE; + } + + ret = copy_to_user((void *)udrv_data->out_addr, &resp, + min(udrv_data->out_len, (uint32_t)sizeof(resp))); + if (ret) + dev_err(dev->dev, + "copy ctx resp to user failed, ret = %d.\n", ret); + + return ret; +} + +static void udma_uar_free(struct udma_dev *udma_dev, + struct udma_ucontext *context) +{ + ida_free(&udma_dev->uar_ida.ida, (int)context->uar.logic_idx); +} + +static struct ubcore_ucontext *udma_alloc_ucontext(struct ubcore_device *dev, + uint32_t uasid, + struct ubcore_udrv_priv *udrv_data) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + struct udma_ucontext *context; + int ret; + + context = kzalloc(sizeof(struct udma_ucontext), GFP_KERNEL); + if (!context) + return NULL; + + ret = udma_uar_alloc(udma_dev, &context->uar); + if (ret) { + dev_err(udma_dev->dev, "Alloc udma_uar Failed.\n"); + goto err_alloc_ucontext; + } + + ret = udma_register_udca(udma_dev, context, udrv_data); + if (ret) { + dev_err(udma_dev->dev, "Register udca Failed.\n"); + goto err_alloc_uar; + } + + context->pdn = uasid; /* Use the UASID as pd number */ + ret = udma_init_ctx_resp(udma_dev, udrv_data, &context->dca_ctx); + if (ret) { + dev_err(udma_dev->dev, "Init ctx resp failed.\n"); + goto err_alloc_uar; + } + + return &context->uctx; + +err_alloc_uar: + udma_uar_free(udma_dev, context); +err_alloc_ucontext: + kfree(context); + return NULL; +} + +static int udma_free_ucontext(struct ubcore_ucontext *uctx) +{ + struct udma_ucontext *context = to_udma_ucontext(uctx); + struct udma_dev *udma_dev = to_udma_dev(uctx->ub_dev); + + if (udma_dev->caps.flags & UDMA_CAP_FLAG_DCA_MODE) + udma_unregister_udca(udma_dev, context); + + ida_free(&udma_dev->uar_ida.ida, (int)context->uar.logic_idx); + kfree(context); + return 0; +} + +static int get_mmap_cmd(struct vm_area_struct *vma) +{ + return (vma->vm_pgoff & MAP_COMMAND_MASK); +} + +static uint64_t get_mmap_idx(struct vm_area_struct *vma) +{ + return ((vma->vm_pgoff >> MAP_INDEX_SHIFT) & MAP_INDEX_MASK); +} + +static int mmap_dca(struct ubcore_ucontext *context, struct vm_area_struct *vma) +{ + struct udma_ucontext *uctx = to_udma_ucontext(context); + struct udma_dca_ctx *ctx = &uctx->dca_ctx; + struct page **pages; + unsigned long num; + int ret; + + if (vma->vm_end - vma->vm_start != (ctx->status_npage * PAGE_SIZE)) + return -EINVAL; + + if (!(vma->vm_flags & VM_WRITE) || + !(vma->vm_flags & VM_SHARED) || + (vma->vm_flags & VM_EXEC)) + return -EPERM; + + if (!ctx->buf_status) + return -EOPNOTSUPP; + + pages = kcalloc(ctx->status_npage, sizeof(struct page *), GFP_KERNEL); + if (!pages) + return -ENOMEM; + + for (num = 0; num < ctx->status_npage; num++) + pages[num] = virt_to_page(ctx->buf_status + num * PAGE_SIZE); + + ret = vm_insert_pages(vma, vma->vm_start, pages, &num); + kfree(pages); + + return ret; +} + +static int udma_mmap(struct ubcore_ucontext *uctx, struct vm_area_struct *vma) +{ + struct udma_dev *udma_dev = to_udma_dev(uctx->ub_dev); + uint64_t address; + uint64_t qpn; + int cmd; + + if (((vma->vm_end - vma->vm_start) % PAGE_SIZE) != 0) { + dev_err(udma_dev->dev, + "mmap failed, unexpected vm area size.\n"); + return -EINVAL; + } + + cmd = get_mmap_cmd(vma); + switch (cmd) { + case UDMA_MMAP_UAR_PAGE: + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + if (io_remap_pfn_range(vma, vma->vm_start, + to_udma_ucontext(uctx)->uar.pfn, + PAGE_SIZE, vma->vm_page_prot)) + return -EAGAIN; + break; + case UDMA_MMAP_DWQE_PAGE: + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + qpn = get_mmap_idx(vma); + address = udma_dev->dwqe_page + qpn * UDMA_DWQE_PAGE_SIZE; + if (io_remap_pfn_range(vma, vma->vm_start, + address >> PAGE_SHIFT, + UDMA_DWQE_PAGE_SIZE, vma->vm_page_prot)) + return -EAGAIN; + break; + case UDMA_MMAP_RESET_PAGE: + if (vma->vm_flags & (VM_WRITE | VM_EXEC)) + return -EINVAL; + + if (remap_pfn_range(vma, vma->vm_start, + page_to_pfn(udma_dev->reset_page), + PAGE_SIZE, vma->vm_page_prot)) + return -EAGAIN; + break; + case UDMA_MMAP_TYPE_DCA: + if (mmap_dca(uctx, vma)) + return -EAGAIN; + break; + default: + dev_err(udma_dev->dev, + "mmap failed, cmd(%d) not support\n", cmd); + return -EINVAL; + } + + return 0; +} + +static int udma_query_stats(const struct ubcore_device *dev, struct ubcore_stats_key *key, + struct ubcore_stats_val *val) +{ + struct ubcore_stats_com_val *com_val = (struct ubcore_stats_com_val *)val->addr; + struct udma_cmq_desc desc[UDMA_QUERY_COUNTER]; + struct udma_dev *udma_dev = to_udma_dev(dev); + struct udma_tx_err_cnt_cmd_data *resp_tx_err; + struct udma_rx_cnt_cmd_data *resp_rx; + struct udma_tx_cnt_cmd_data *resp_tx; + int ret; + int i; + + if (val->len != sizeof(struct ubcore_stats_com_val)) { + dev_err(udma_dev->dev, "The val len is err.\n"); + return -EINVAL; + } + + for (i = 0; i < UDMA_QUERY_COUNTER; i++) { + udma_cmq_setup_basic_desc(&desc[i], UDMA_OPC_QUERY_COUNTER, true); + if (i < (UDMA_QUERY_COUNTER - 1)) + desc[i].flag |= cpu_to_le16(UDMA_CMD_FLAG_NEXT); + else + desc[i].flag &= ~cpu_to_le16(UDMA_CMD_FLAG_NEXT); + } + + ret = udma_cmq_send(udma_dev, desc, UDMA_QUERY_COUNTER); + if (ret) { + dev_err(udma_dev->dev, "Failed to query stats, ret = %d.\n", ret); + return ret; + } + + resp_rx = (struct udma_rx_cnt_cmd_data *)desc[UDMA_QX_RESP].data; + resp_tx = (struct udma_tx_cnt_cmd_data *)desc[UDMA_TX_RESP].data; + resp_tx_err = (struct udma_tx_err_cnt_cmd_data *)desc[UDMA_TX_ERR_RESP].data; + + com_val->tx_pkt = resp_tx->pkt_tx_cnt; + com_val->tx_pkt_err = resp_tx_err->err_pkt_tx_cnt; + + com_val->rx_pkt = resp_rx->pkt_rx_cnt; + com_val->rx_pkt_err = resp_rx->err_pkt_rx_cnt; + + /* tx_bytes and rx_bytes are not support now */ + com_val->tx_bytes = 0; + com_val->rx_bytes = 0; + + return ret; +} + +static uint16_t query_congest_alg(uint8_t udma_cc_caps) +{ + uint16_t ubcore_cc_alg = 0; + + if (udma_cc_caps & UDMA_CONG_SEL_DCQCN) + ubcore_cc_alg |= UBCORE_CC_DCQCN; + if (udma_cc_caps & UDMA_CONG_SEL_LDCP) + ubcore_cc_alg |= UBCORE_CC_LDCP; + if (udma_cc_caps & UDMA_CONG_SEL_HC3) + ubcore_cc_alg |= UBCORE_CC_HC3; + if (udma_cc_caps & UDMA_CONG_SEL_DIP) + ubcore_cc_alg |= UBCORE_CC_DIP; + + return ubcore_cc_alg; +} + +static int udma_query_device_attr(struct ubcore_device *dev, + struct ubcore_device_attr *attr) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + struct device *dev_of_udma = udma_dev->dev; + struct net_device *net_dev; + int i; + + attr->guid = udma_dev->sys_image_guid; + attr->dev_cap.max_jfc = (1 << udma_dev->caps.num_jfc_shift); + attr->dev_cap.max_jfs = (1 << udma_dev->caps.num_jfs_shift); + attr->dev_cap.max_jfr = (1 << udma_dev->caps.num_jfr_shift); + attr->dev_cap.max_jetty = (1 << udma_dev->caps.num_jetty_shift); + attr->dev_cap.max_jfc_depth = udma_dev->caps.max_cqes; + attr->dev_cap.max_jfs_depth = udma_dev->caps.max_wqes; + attr->dev_cap.max_jfr_depth = udma_dev->caps.max_srq_wrs; + attr->dev_cap.max_jfs_inline_size = udma_dev->caps.max_sq_inline; + attr->dev_cap.max_jfs_sge = udma_dev->caps.max_sq_sg; + attr->dev_cap.max_jfr_sge = udma_dev->caps.max_srq_sges; + attr->dev_cap.max_msg_size = UDMA_MAX_MSG_LEN; + attr->dev_cap.trans_mode = UBCORE_TP_RM | UBCORE_TP_UM; + attr->dev_cap.feature.bs.oor = udma_dev->caps.oor_en; + attr->port_cnt = udma_dev->caps.num_ports; + attr->dev_cap.comp_vector_cnt = udma_dev->caps.num_comp_vectors; + attr->vf_cnt = udma_dev->func_num - 1; + attr->dev_cap.feature.bs.jfc_inline = !!(udma_dev->caps.flags & UDMA_CAP_FLAG_CQE_INLINE); + attr->dev_cap.feature.bs.spray_en = 1; + attr->dev_cap.max_jfs_rsge = udma_dev->caps.max_sq_sg; + attr->dev_cap.congestion_ctrl_alg = query_congest_alg(udma_dev->caps.cong_type); + + for (i = 0; i < udma_dev->caps.num_ports; i++) { + net_dev = udma_dev->uboe.netdevs[i]; + if (!net_dev) { + dev_err(dev_of_udma, "Find netdev %u failed!\n", i); + return -EINVAL; + } + attr->port_attr[i].max_mtu = + udma_mtu_int_to_enum(net_dev->max_mtu); + } + + return 0; +} + +static int udma_get_active_speed(uint32_t speed, struct ubcore_port_status *port_status) +{ + if (speed == SPEED_100G) { + port_status->active_width = UBCORE_LINK_X1; + port_status->active_speed = UBCORE_SP_100G; + } else if (speed == SPEED_200G) { + port_status->active_width = UBCORE_LINK_X1; + port_status->active_speed = UBCORE_SP_200G; + } else { + return -EINVAL; + } + + return 0; +} + +static int udma_query_device_status(const struct ubcore_device *dev, + struct ubcore_device_status *dev_status) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + enum ubcore_mtu net_dev_mtu; + struct net_device *net_dev; + enum ubcore_mtu mtu; + uint8_t port_num; + int ret; + int i; + + port_num = udma_dev->caps.num_ports; + + for (i = 0; i < port_num; i++) { + net_dev = udma_dev->uboe.netdevs[i]; + if (!net_dev) { + dev_err(udma_dev->dev, "Find netdev %u failed!\n", i); + return -EINVAL; + } + + dev_status->port_status[i].state = + (netif_running(net_dev) && + netif_carrier_ok(net_dev)) ? + UBCORE_PORT_ACTIVE : UBCORE_PORT_DOWN; + net_dev_mtu = ubcore_get_mtu(net_dev->max_mtu); + mtu = ubcore_get_mtu(net_dev->mtu); + + dev_status->port_status[i].active_mtu = (enum ubcore_mtu) + (mtu ? min(net_dev_mtu, mtu) : UBCORE_MTU_256); + + ret = udma_get_active_speed(udma_dev->caps.speed, &dev_status->port_status[i]); + if (ret) { + dev_err(udma_dev->dev, "Port[%u] query speed and width failed!\n", i); + return ret; + } + } + + return 0; +} + +int udma_user_ctl_flush_cqe(struct ubcore_ucontext *uctx, struct ubcore_user_ctl_in *in, + struct ubcore_user_ctl_out *out, + struct ubcore_udrv_priv *udrv_data) +{ + struct udma_dev *udma_device = to_udma_dev(uctx->ub_dev); + struct flush_cqe_param fcp; + struct udma_qp *udma_qp; + uint32_t sq_pi; + uint32_t qpn; + int ret; + + ret = (int)copy_from_user(&fcp, (void *)in->addr, + sizeof(struct flush_cqe_param)); + if (ret != 0) { + dev_err(udma_device->dev, + "copy_from_user failed in flush_cqe, ret:%d.\n", ret); + return -EFAULT; + } + sq_pi = fcp.sq_producer_idx; + qpn = fcp.qpn; + + xa_lock(&udma_device->qp_table.xa); + udma_qp = (struct udma_qp *)xa_load(&udma_device->qp_table.xa, qpn); + if (!udma_qp) { + dev_err(udma_device->dev, "get qp(0x%x) error.\n", qpn); + xa_unlock(&udma_device->qp_table.xa); + return -EINVAL; + } + refcount_inc(&udma_qp->refcount); + xa_unlock(&udma_device->qp_table.xa); + + ret = udma_flush_cqe(udma_device, udma_qp, sq_pi); + + if (refcount_dec_and_test(&udma_qp->refcount)) + complete(&udma_qp->free); + + return ret; +} + +static int config_poe_addr(struct udma_dev *udma_device, uint8_t id, + uint64_t addr) +{ + struct udma_poe_cfg_addr_cmq *cmd; + struct udma_cmq_desc desc; + int ret; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_CFG_POE_ADDR, false); + cmd = (struct udma_poe_cfg_addr_cmq *)desc.data; + cmd->channel_id = cpu_to_le32(id); + cmd->poe_addr_l = cpu_to_le32(lower_32_bits(addr)); + cmd->poe_addr_h = cpu_to_le32(upper_32_bits(addr)); + + ret = udma_cmq_send(udma_device, &desc, 1); + if (ret) + dev_err(udma_device->dev, + "configure poe channel %u addr failed, ret = %d.\n", + id, ret); + return ret; +} + +static int config_poe_attr(struct udma_dev *udma_device, uint8_t id, bool en) +{ + struct udma_poe_cfg_attr_cmq *cmd; + struct udma_cmq_desc desc; + int ret; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_CFG_POE_ATTR, false); + cmd = (struct udma_poe_cfg_attr_cmq *)desc.data; + cmd->channel_id = cpu_to_le32(id); + cmd->rsv_en_outstd = en ? 1 : 0; + + ret = udma_cmq_send(udma_device, &desc, 1); + if (ret) + dev_err(udma_device->dev, + "configure poe channel %u attr failed, ret = %d.\n", + id, ret); + return ret; +} + +static int check_poe_channel(struct udma_dev *udma_device, uint8_t poe_ch) +{ + if (poe_ch >= udma_device->caps.poe_ch_num) { + dev_err(udma_device->dev, "invalid POE channel %u.\n", poe_ch); + return -EINVAL; + } + + return 0; +} + +int udma_user_ctl_config_poe(struct ubcore_ucontext *uctx, struct ubcore_user_ctl_in *in, + struct ubcore_user_ctl_out *out, + struct ubcore_udrv_priv *udrv_data) +{ + struct udma_poe_info poe_info; + struct udma_dev *udma_device; + int ret; + + udma_device = to_udma_dev(uctx->ub_dev); + ret = (int)copy_from_user(&poe_info, + (void *)in->addr, + sizeof(struct udma_poe_info)); + if (ret) { + dev_err(udma_device->dev, "cp from user failed in config poe, ret:%d.\n", + ret); + return -EFAULT; + } + + ret = check_poe_channel(udma_device, poe_info.poe_channel); + if (ret) { + dev_err(udma_device->dev, "check channel failed in config poe, ret:%d.\n", + ret); + return ret; + } + + ret = config_poe_attr(udma_device, poe_info.poe_channel, + !!poe_info.poe_addr); + if (ret) { + dev_err(udma_device->dev, "config attr failed in config poe, ret:%d.\n", + ret); + config_poe_addr(udma_device, poe_info.poe_channel, 0); + return ret; + } + + ret = config_poe_addr(udma_device, poe_info.poe_channel, + poe_info.poe_addr); + if (ret) + dev_err(udma_device->dev, "config addr failed in config poe, ret:%d.\n", + ret); + + return ret; +} + +static int query_poe_addr(struct udma_dev *udma_device, uint8_t id, + uint64_t *addr) +{ +#define POE_ADDR_H_SHIFT 32 + struct udma_poe_cfg_addr_cmq *resp; + struct udma_cmq_desc desc; + int ret; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_CFG_POE_ADDR, true); + resp = (struct udma_poe_cfg_addr_cmq *)desc.data; + resp->channel_id = cpu_to_le32(id); + + ret = udma_cmq_send(udma_device, &desc, 1); + if (ret) { + dev_err(udma_device->dev, + "Query poe channel %u addr failed, ret = %d.\n", + id, ret); + return ret; + } + + *addr = resp->poe_addr_l | ((uint64_t)resp->poe_addr_h << + POE_ADDR_H_SHIFT); + + return ret; +} + +static int query_poe_attr(struct udma_dev *udma_device, uint8_t id, bool *en) +{ + struct udma_poe_cfg_attr_cmq *resp; + struct udma_cmq_desc desc; + int ret; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_CFG_POE_ATTR, true); + resp = (struct udma_poe_cfg_attr_cmq *)desc.data; + resp->channel_id = cpu_to_le32(id); + + ret = udma_cmq_send(udma_device, &desc, 1); + if (ret) { + dev_err(udma_device->dev, + "Query poe channel %u attr failed, ret = %d.\n", + id, ret); + return ret; + } + + *en = !!resp->rsv_en_outstd; + + return ret; +} + +int udma_user_ctl_query_poe(struct ubcore_ucontext *uctx, struct ubcore_user_ctl_in *in, + struct ubcore_user_ctl_out *out, + struct ubcore_udrv_priv *udrv_data) +{ + struct udma_poe_info poe_info_out; + struct udma_poe_info poe_info_in; + struct udma_dev *udma_device; + uint64_t poe_addr; + bool poe_en; + int ret; + + udma_device = to_udma_dev(uctx->ub_dev); + ret = (int)copy_from_user(&poe_info_in, (void *)in->addr, + sizeof(struct udma_poe_info)); + if (ret) { + dev_err(udma_device->dev, "cp from user failed in query poe, ret:%d.\n", + ret); + return -EFAULT; + } + + ret = check_poe_channel(udma_device, poe_info_in.poe_channel); + if (ret) { + dev_err(udma_device->dev, "check channel failed in query poe, ret:%d.\n", + ret); + return ret; + } + + ret = query_poe_attr(udma_device, poe_info_in.poe_channel, &poe_en); + if (ret) { + dev_err(udma_device->dev, "query attr failed in query poe, ret:%d.\n", + ret); + return ret; + } + + ret = query_poe_addr(udma_device, poe_info_in.poe_channel, &poe_addr); + if (ret) { + dev_err(udma_device->dev, "query addr failed in query poe, ret:%d.\n", + ret); + return ret; + } + + poe_info_out.en = poe_en ? 1 : 0; + poe_info_out.poe_addr = poe_addr; + ret = (int)copy_to_user((void *)out->addr, &poe_info_out, + min(out->len, + (uint32_t)sizeof(struct udma_poe_info))); + if (ret != 0) { + dev_err(udma_device->dev, "cp to user failed in query poe, ret:%d.\n", + ret); + return -EFAULT; + } + return ret; +} + +int udma_user_ctl_dca_reg(struct ubcore_ucontext *uctx, struct ubcore_user_ctl_in *in, + struct ubcore_user_ctl_out *out, struct ubcore_udrv_priv *udrv_data) +{ + struct udma_dev *udma_device = to_udma_dev(uctx->ub_dev); + struct udma_ucontext *context = to_udma_ucontext(uctx); + struct udma_dca_reg_attr attr = {}; + int ret; + + ret = (int)copy_from_user(&attr, (void *)in->addr, + sizeof(struct udma_dca_reg_attr)); + if (ret) { + dev_err(udma_device->dev, "cp from user failed in dca reg, ret:%d.\n", + ret); + return -EFAULT; + } + + ret = udma_register_dca_mem(udma_device, context, &attr); + if (ret) { + dev_err(udma_device->dev, + "register dca mem failed, ret:%d.\n", ret); + return ret; + } + + return 0; +} + +int udma_user_ctl_dca_dereg(struct ubcore_ucontext *uctx, struct ubcore_user_ctl_in *in, + struct ubcore_user_ctl_out *out, + struct ubcore_udrv_priv *udrv_data) +{ + struct udma_dev *udma_device = to_udma_dev(uctx->ub_dev); + struct udma_ucontext *context = to_udma_ucontext(uctx); + struct udma_dca_dereg_attr attr = {}; + int ret; + + ret = (int)copy_from_user(&attr, (void *)in->addr, + sizeof(struct udma_dca_dereg_attr)); + if (ret) { + dev_err(udma_device->dev, "cp from user failed in dca dereg, ret:%d.\n", + ret); + return -EFAULT; + } + + attr.mem = NULL; + ret = udma_deregister_dca_mem(udma_device, context, &attr, true); + if (ret) { + dev_err(udma_device->dev, "deregister dca mem failed, ret:%d.\n", ret); + return -EFAULT; + } + + return 0; +} + +int udma_user_ctl_dca_shrink(struct ubcore_ucontext *uctx, struct ubcore_user_ctl_in *in, + struct ubcore_user_ctl_out *out, + struct ubcore_udrv_priv *udrv_data) +{ + struct udma_dev *udma_device = to_udma_dev(uctx->ub_dev); + struct udma_ucontext *context = to_udma_ucontext(uctx); + struct udma_dca_shrink_attr shrink_attr = {}; + struct udma_dca_shrink_resp shrink_resp = {}; + struct udma_dca_dereg_attr dereg_attr = {}; + int ret; + + ret = (int)copy_from_user(&shrink_attr, (void *)in->addr, + sizeof(struct udma_dca_shrink_attr)); + if (ret) { + dev_err(udma_device->dev, "cp from user failed in dca shrink, ret:%d.\n", + ret); + return -EFAULT; + } + + udma_shrink_dca_mem(udma_device, context, &shrink_attr, &shrink_resp); + + if (shrink_resp.free_mems >= 1) { + dereg_attr.mem = shrink_resp.mem; + udma_deregister_dca_mem(udma_device, context, &dereg_attr, false); + shrink_resp.mem = NULL; + } + + ret = (int)copy_to_user((void *)out->addr, &shrink_resp, + min(out->len, + (uint32_t)sizeof(struct udma_dca_shrink_resp))); + if (ret) { + dev_err(udma_device->dev, "cp to user failed in dca_shrink, ret:%d.\n", + ret); + return -EFAULT; + } + + return 0; +} + +int udma_user_ctl_dca_attach(struct ubcore_ucontext *uctx, struct ubcore_user_ctl_in *in, + struct ubcore_user_ctl_out *out, + struct ubcore_udrv_priv *udrv_data) +{ + struct udma_dev *udma_device = to_udma_dev(uctx->ub_dev); + struct udma_dca_attach_attr attr = {}; + struct udma_dca_attach_resp resp = {}; + int ret; + + ret = (int)copy_from_user(&attr, (void *)in->addr, + sizeof(struct udma_dca_attach_attr)); + if (ret) { + dev_err(udma_device->dev, "cp from user failed in dca attach, ret:%d.\n", + ret); + return -EFAULT; + } + + ret = udma_dca_attach(udma_device, &attr, &resp); + if (ret) { + dev_err(udma_device->dev, "attach dca mem failed, ret:%d.\n", + ret); + return ret; + } + + ret = (int)copy_to_user((void *)out->addr, &resp, + min(out->len, + (uint32_t)sizeof(struct udma_dca_attach_resp))); + if (ret) { + udma_dca_disattach(udma_device, &attr); + dev_err(udma_device->dev, "copy_to_user failed in dca_attach, ret:%d.\n", + ret); + return -EFAULT; + } + + return 0; +} + +int udma_user_ctl_dca_detach(struct ubcore_ucontext *uctx, struct ubcore_user_ctl_in *in, + struct ubcore_user_ctl_out *out, + struct ubcore_udrv_priv *udrv_data) +{ + struct udma_dev *udma_device = to_udma_dev(uctx->ub_dev); + struct udma_dca_detach_attr attr = {}; + int ret; + + ret = (int)copy_from_user(&attr, (void *)in->addr, + sizeof(struct udma_dca_detach_attr)); + if (ret) { + dev_err(udma_device->dev, "cp from user failed in dca detach, ret:%d.\n", + ret); + return -EFAULT; + } + + udma_dca_detach(udma_device, &attr); + + return 0; +} + +int udma_user_ctl_dca_query(struct ubcore_ucontext *uctx, struct ubcore_user_ctl_in *in, + struct ubcore_user_ctl_out *out, + struct ubcore_udrv_priv *udrv_data) +{ + struct udma_dev *udma_device = to_udma_dev(uctx->ub_dev); + struct udma_dca_query_attr attr = {}; + struct udma_dca_query_resp resp = {}; + int ret; + + ret = (int)copy_from_user(&attr, (void *)in->addr, + sizeof(struct udma_dca_query_attr)); + if (ret) { + dev_err(udma_device->dev, "copy_from_user failed in dca query, ret:%d.\n", + ret); + return -EFAULT; + } + + ret = udma_query_dca_mem(udma_device, &attr, &resp); + if (ret) { + dev_err(udma_device->dev, "query dca mem failed, ret:%d.\n", + ret); + return ret; + } + + ret = (int)copy_to_user((void *)out->addr, &resp, + min(out->len, + (uint32_t)sizeof(struct udma_dca_query_resp))); + if (ret) { + dev_err(udma_device->dev, "cp to user failed in dca_query, ret:%d.\n", + ret); + return -EFAULT; + } + + return 0; +} + +typedef int (*udma_user_ctl_opcode)(struct ubcore_ucontext *uctx, + struct ubcore_user_ctl_in *in, + struct ubcore_user_ctl_out *out, + struct ubcore_udrv_priv *udrv_data); + +static udma_user_ctl_opcode g_udma_user_ctl_opcodes[] = { + [UDMA_USER_CTL_FLUSH_CQE] = udma_user_ctl_flush_cqe, + [UDMA_CONFIG_POE_CHANNEL] = udma_user_ctl_config_poe, + [UDMA_QUERY_POE_CHANNEL] = udma_user_ctl_query_poe, + [UDMA_DCA_MEM_REG] = udma_user_ctl_dca_reg, + [UDMA_DCA_MEM_DEREG] = udma_user_ctl_dca_dereg, + [UDMA_DCA_MEM_SHRINK] = udma_user_ctl_dca_shrink, + [UDMA_DCA_MEM_ATTACH] = udma_user_ctl_dca_attach, + [UDMA_DCA_MEM_DETACH] = udma_user_ctl_dca_detach, + [UDMA_DCA_MEM_QUERY] = udma_user_ctl_dca_query, +}; + +int udma_user_ctl(struct ubcore_user_ctl *k_user_ctl) +{ + struct ubcore_udrv_priv udrv_data = k_user_ctl->udrv_data; + struct ubcore_user_ctl_out out = k_user_ctl->out; + struct ubcore_ucontext *uctx = k_user_ctl->uctx; + struct ubcore_user_ctl_in in = k_user_ctl->in; + struct udma_dev *udma_device; + + udma_device = to_udma_dev(uctx->ub_dev); + if (in.opcode >= UDMA_OPCODE_NUM || + !g_udma_user_ctl_opcodes[in.opcode]) { + dev_err(udma_device->dev, "bad user_ctl opcode: 0x%x.\n", + (int)in.opcode); + return -EINVAL; + } + return g_udma_user_ctl_opcodes[in.opcode](uctx, &in, &out, &udrv_data); +} + +static struct ubcore_ops g_udma_dev_ops = { + .owner = THIS_MODULE, + .abi_version = 1, + .set_eid = udma_set_eid, + .query_device_attr = udma_query_device_attr, + .query_device_status = udma_query_device_status, + .query_res = udma_query_res, + .alloc_ucontext = udma_alloc_ucontext, + .free_ucontext = udma_free_ucontext, + .mmap = udma_mmap, + .register_seg = udma_register_seg, + .unregister_seg = udma_unregister_seg, + .import_seg = udma_import_seg, + .unimport_seg = udma_unimport_seg, + .create_jfc = udma_create_jfc, + .modify_jfc = udma_modify_jfc, + .destroy_jfc = udma_destroy_jfc, + .create_jfs = udma_create_jfs, + .destroy_jfs = udma_destroy_jfs, + .create_jfr = udma_create_jfr, + .modify_jfr = udma_modify_jfr, + .destroy_jfr = udma_destroy_jfr, + .import_jfr = udma_import_jfr, + .unimport_jfr = udma_unimport_jfr, + .create_jetty = udma_create_jetty, + .destroy_jetty = udma_destroy_jetty, + .import_jetty = udma_import_jetty, + .unimport_jetty = udma_unimport_jetty, + .create_tp = udma_create_tp, + .modify_tp = udma_modify_tp, + .destroy_tp = udma_destroy_tp, + .user_ctl = udma_user_ctl, + .query_stats = udma_query_stats, +}; + +static void udma_cleanup_uar_table(struct udma_dev *dev) +{ + struct udma_ida *uar_ida = &dev->uar_ida; + + if (!ida_is_empty(&uar_ida->ida)) + dev_err(dev->dev, "IDA not empty in clean up uar table.\n"); + ida_destroy(&uar_ida->ida); +} + +static void udma_init_uar_table(struct udma_dev *udma_dev) +{ + struct udma_ida *uar_ida = &udma_dev->uar_ida; + + ida_init(&uar_ida->ida); + uar_ida->max = udma_dev->caps.num_uars - 1; + uar_ida->min = udma_dev->caps.reserved_uars; +} + +static void udma_cleanup_seg_table(struct udma_dev *dev) +{ + struct udma_ida *seg_ida = &dev->seg_table.seg_ida; + + if (!ida_is_empty(&seg_ida->ida)) + dev_err(dev->dev, "IDA not empty in clean up seg table.\n"); + ida_destroy(&seg_ida->ida); +} + +static void udma_init_seg_table(struct udma_dev *udma_dev) +{ + struct udma_ida *seg_ida = &udma_dev->seg_table.seg_ida; + + ida_init(&seg_ida->ida); + seg_ida->max = udma_dev->caps.num_mtpts - 1; + seg_ida->min = udma_dev->caps.reserved_mrws; +} + +static void udma_cleanup_jfc_table(struct udma_dev *udma_dev) +{ + struct udma_jfc_table *jfc_table = &udma_dev->jfc_table; + struct udma_jfc *jfc; + unsigned long index; + int i; + + for (i = 0; i < UDMA_CQ_BANK_NUM; i++) { + if (!ida_is_empty(&jfc_table->bank[i].ida)) + dev_err(udma_dev->dev, + "IDA not empty in clean up jfc bank[%d] table\n", + i); + ida_destroy(&jfc_table->bank[i].ida); + } + + if (!xa_empty(&jfc_table->xa)) { + dev_err(udma_dev->dev, "JFC not empty\n"); + xa_for_each(&jfc_table->xa, index, jfc) + udma_table_put(udma_dev, &jfc_table->table, index); + } + xa_destroy(&jfc_table->xa); +} + +static void udma_init_jfc_table(struct udma_dev *udma_dev) +{ + struct udma_jfc_table *jfc_table = &udma_dev->jfc_table; + uint32_t reserved_from_bot; + uint32_t i; + + mutex_init(&jfc_table->bank_mutex); + xa_init(&jfc_table->xa); + + reserved_from_bot = 1; + + for (i = 0; i < reserved_from_bot; i++) { + jfc_table->bank[get_jfc_bankid(i)].inuse++; + jfc_table->bank[get_jfc_bankid(i)].min++; + } + + for (i = 0; i < UDMA_CQ_BANK_NUM; i++) { + ida_init(&jfc_table->bank[i].ida); + jfc_table->bank[i].max = (1 << udma_dev->caps.num_jfc_shift) / + UDMA_CQ_BANK_NUM - 1; + } +} + +static void udma_cleanup_jfr_table(struct udma_dev *dev) +{ + struct udma_jfr_table *jfr_table = &dev->jfr_table; + struct udma_ida *jfr_ida = &jfr_table->jfr_ida; + unsigned long index = 0; + struct udma_jfr *jfr; + + if (!ida_is_empty(&jfr_ida->ida)) + dev_err(dev->dev, "IDA not empty in clean up jfr table.\n"); + ida_destroy(&jfr_ida->ida); + + if (!xa_empty(&jfr_table->xa)) { + dev_err(dev->dev, "JFR not empty\n"); + xa_for_each(&jfr_table->xa, index, jfr) + udma_table_put(dev, &jfr_table->table, index); + } + xa_destroy(&jfr_table->xa); +} + +static void udma_init_jfr_table(struct udma_dev *dev) +{ + struct udma_jfr_table *jfr_table = &dev->jfr_table; + struct udma_ida *jfr_ida = &jfr_table->jfr_ida; + + xa_init(&jfr_table->xa); + ida_init(&jfr_ida->ida); + jfr_ida->max = (1 << dev->caps.num_jfr_shift) - 1; + /* reserve jfr id 0 */ + jfr_ida->min = 1; +} + +static void udma_cleanup_jfs_table(struct udma_dev *dev) +{ + struct udma_jfs_table *jfs_table = &dev->jfs_table; + struct udma_ida *jfs_ida = &jfs_table->jfs_ida; + + if (!ida_is_empty(&jfs_ida->ida)) + dev_err(dev->dev, "IDA not empty in clean up jfs table.\n"); + ida_destroy(&jfs_ida->ida); + + if (!xa_empty(&jfs_table->xa)) + dev_err(dev->dev, "JFS table not empty.\n"); + xa_destroy(&jfs_table->xa); +} + +static void udma_init_jfs_table(struct udma_dev *dev) +{ + struct udma_jfs_table *jfs_table = &dev->jfs_table; + struct udma_ida *jfs_ida = &jfs_table->jfs_ida; + + xa_init(&jfs_table->xa); + ida_init(&jfs_ida->ida); + jfs_ida->max = (1 << dev->caps.num_jfs_shift) - 1; + /* reserve jfs id 0 */ + jfs_ida->min = 1; +} + +static void udma_cleanup_jetty_table(struct udma_dev *dev) +{ + struct udma_jetty_table *jetty_table = &dev->jetty_table; + struct udma_ida *jetty_ida = &jetty_table->jetty_ida; + + if (!ida_is_empty(&jetty_ida->ida)) + dev_err(dev->dev, "IDA not empty in clean up jetty table.\n"); + ida_destroy(&jetty_ida->ida); + + if (!xa_empty(&jetty_table->xa)) + dev_err(dev->dev, "Jetty table not empty.\n"); + xa_destroy(&jetty_table->xa); +} + +static void udma_init_jetty_table(struct udma_dev *dev) +{ + struct udma_jetty_table *jetty_table = &dev->jetty_table; + struct udma_ida *jetty_ida = &jetty_table->jetty_ida; + + xa_init(&jetty_table->xa); + ida_init(&jetty_ida->ida); + jetty_ida->max = (1 << dev->caps.num_jetty_shift) - 1; + /* reserve jetty id 0 */ + jetty_ida->min = 1; +} + +int udma_init_eq_idx_table(struct udma_dev *udma_dev) +{ + uint32_t eq_num; + + eq_num = udma_dev->caps.num_comp_vectors + + udma_dev->caps.num_aeq_vectors; + udma_dev->eq_table.idx_table = kcalloc(eq_num, sizeof(uint32_t), + GFP_KERNEL); + if (ZERO_OR_NULL_PTR(udma_dev->eq_table.idx_table)) + return -ENOMEM; + + return 0; +} + +int udma_setup_hca(struct udma_dev *udma_dev) +{ + struct device *dev = udma_dev->dev; + int ret; + + INIT_LIST_HEAD(&udma_dev->qp_list); + spin_lock_init(&udma_dev->qp_list_lock); + INIT_LIST_HEAD(&udma_dev->dip_list); + spin_lock_init(&udma_dev->dip_list_lock); + + if (udma_dev->caps.flags & UDMA_CAP_FLAG_CQ_RECORD_DB || + udma_dev->caps.flags & UDMA_CAP_FLAG_QP_RECORD_DB) { + INIT_LIST_HEAD(&udma_dev->pgdir_list); + mutex_init(&udma_dev->pgdir_mutex); + } + + udma_init_uar_table(udma_dev); + + ret = udma_init_qp_table(udma_dev); + if (ret) { + dev_err(dev, "Failed to init qp_table.\n"); + goto err_uar_table_free; + } + + udma_init_seg_table(udma_dev); + udma_init_jfc_table(udma_dev); + udma_init_jfr_table(udma_dev); + udma_init_jfs_table(udma_dev); + udma_init_jetty_table(udma_dev); + ret = udma_init_eq_idx_table(udma_dev); + if (ret) { + dev_err(dev, "Failed to init eq_table.\n"); + goto err_eq_table; + } + + return 0; +err_eq_table: + udma_cleanup_jetty_table(udma_dev); + udma_cleanup_jfs_table(udma_dev); + udma_cleanup_jfr_table(udma_dev); + udma_cleanup_jfc_table(udma_dev); + udma_cleanup_seg_table(udma_dev); + udma_cleanup_qp_table(udma_dev); + +err_uar_table_free: + udma_cleanup_uar_table(udma_dev); + return ret; +} + +void udma_teardown_hca(struct udma_dev *udma_dev) +{ + kfree(udma_dev->eq_table.idx_table); + udma_cleanup_jetty_table(udma_dev); + udma_cleanup_jfs_table(udma_dev); + udma_cleanup_jfr_table(udma_dev); + udma_cleanup_jfc_table(udma_dev); + udma_cleanup_seg_table(udma_dev); + udma_cleanup_qp_table(udma_dev); + udma_cleanup_uar_table(udma_dev); +} + +int udma_init_common_hem(struct udma_dev *udma_dev) +{ + struct device *dev = udma_dev->dev; + int ret; + + ret = udma_init_hem_table(udma_dev, &udma_dev->seg_table.table, + HEM_TYPE_MTPT, udma_dev->caps.mtpt_entry_sz, + udma_dev->caps.num_mtpts); + if (ret) { + dev_err(dev, "Failed to init MTPT context memory.\n"); + return ret; + } + dev_info(dev, "init MPT hem table success.\n"); + + ret = udma_init_hem_table(udma_dev, &udma_dev->qp_table.qp_table, + HEM_TYPE_QPC, udma_dev->caps.qpc_sz, + udma_dev->caps.num_qps); + if (ret) { + dev_err(dev, "Failed to init QP context memory.\n"); + goto err_unmap_dmpt; + } + dev_info(dev, "init QPC hem table success.\n"); + + ret = udma_init_hem_table(udma_dev, &udma_dev->qp_table.irrl_table, + HEM_TYPE_IRRL, udma_dev->caps.irrl_entry_sz * + udma_dev->caps.max_qp_init_rdma, + udma_dev->caps.num_qps); + if (ret) { + dev_err(dev, "Failed to init irrl_table memory.\n"); + goto err_unmap_qp; + } + dev_info(dev, "init IRRL hem table success.\n"); + + if (udma_dev->caps.trrl_entry_sz) { + ret = udma_init_hem_table(udma_dev, + &udma_dev->qp_table.trrl_table, + HEM_TYPE_TRRL, + udma_dev->caps.trrl_entry_sz * + udma_dev->caps.max_qp_dest_rdma, + udma_dev->caps.num_qps); + if (ret) { + dev_err(dev, "Failed to init trrl_table memory.\n"); + goto err_unmap_irrl; + } + dev_info(dev, "init TRRL hem table success.\n"); + } + + ret = udma_init_hem_table(udma_dev, &udma_dev->jfc_table.table, + HEM_TYPE_CQC, udma_dev->caps.cqc_entry_sz, + udma_dev->caps.num_cqs); + if (ret) { + dev_err(dev, "Failed to init CQ context memory.\n"); + goto err_unmap_trrl; + } + dev_info(dev, "init CQC hem table success.\n"); + + if (udma_dev->caps.flags & UDMA_CAP_FLAG_SRQ) { + ret = udma_init_hem_table(udma_dev, &udma_dev->jfr_table.table, + HEM_TYPE_SRQC, + udma_dev->caps.srqc_entry_sz, + udma_dev->caps.num_srqs); + if (ret) { + dev_err(dev, "Failed to init SRQ context memory.\n"); + goto err_unmap_cq; + } + dev_info(dev, "init SRQC hem table success.\n"); + } + + if (udma_dev->caps.flags & UDMA_CAP_FLAG_QP_FLOW_CTRL) { + ret = udma_init_hem_table(udma_dev, + &udma_dev->qp_table.sccc_table, + HEM_TYPE_SCCC, + udma_dev->caps.scc_ctx_sz, + udma_dev->caps.num_qps); + if (ret) { + dev_err(dev, "Failed to init SCC context memory.\n"); + goto err_unmap_srq; + } + dev_info(dev, "init SCCC hem table success.\n"); + } + + if (udma_dev->caps.gmv_entry_sz) { + ret = udma_init_hem_table(udma_dev, &udma_dev->gmv_table, + HEM_TYPE_GMV, + udma_dev->caps.gmv_entry_sz, + udma_dev->caps.gmv_entry_num); + if (ret) { + dev_err(dev, "failed to init gmv table memory.\n"); + goto err_unmap_ctx; + } + dev_info(dev, "init GMV hem table success.\n"); + } + + return 0; +err_unmap_ctx: + if (udma_dev->caps.flags & UDMA_CAP_FLAG_QP_FLOW_CTRL) + udma_cleanup_hem_table(udma_dev, + &udma_dev->qp_table.sccc_table); +err_unmap_srq: + if (udma_dev->caps.flags & UDMA_CAP_FLAG_SRQ) + udma_cleanup_hem_table(udma_dev, &udma_dev->jfr_table.table); +err_unmap_cq: + udma_cleanup_hem_table(udma_dev, &udma_dev->jfc_table.table); +err_unmap_trrl: + if (udma_dev->caps.trrl_entry_sz) + udma_cleanup_hem_table(udma_dev, + &udma_dev->qp_table.trrl_table); +err_unmap_irrl: + udma_cleanup_hem_table(udma_dev, &udma_dev->qp_table.irrl_table); +err_unmap_qp: + udma_cleanup_hem_table(udma_dev, &udma_dev->qp_table.qp_table); +err_unmap_dmpt: + udma_cleanup_hem_table(udma_dev, &udma_dev->seg_table.table); + + return ret; +} + +static int udma_init_hem(struct udma_dev *udma_dev) +{ + struct device *dev = udma_dev->dev; + int ret; + + ret = udma_init_common_hem(udma_dev); + if (ret) { + dev_err(dev, "Failed to init common hem table of PF.\n"); + return ret; + } + + if (udma_dev->caps.qpc_timer_entry_sz) { + ret = udma_init_hem_table(udma_dev, &udma_dev->qpc_timer_table, + HEM_TYPE_QPC_TIMER, + udma_dev->caps.qpc_timer_entry_sz, + udma_dev->caps.num_qpc_timer); + if (ret) { + dev_err(dev, "Failed to init QPC timer memory.\n"); + goto err_unmap_vf_hem; + } + } + if (udma_dev->caps.cqc_timer_entry_sz) { + ret = udma_init_hem_table(udma_dev, &udma_dev->cqc_timer_table, + HEM_TYPE_CQC_TIMER, + udma_dev->caps.cqc_timer_entry_sz, + udma_dev->caps.cqc_timer_bt_num); + if (ret) { + dev_err(dev, "Failed to init CQC timer memory.\n"); + goto err_unmap_qpc_timer; + } + } + + return 0; +err_unmap_qpc_timer: + if (udma_dev->caps.qpc_timer_entry_sz) + udma_cleanup_hem_table(udma_dev, &udma_dev->qpc_timer_table); +err_unmap_vf_hem: + udma_cleanup_common_hem(udma_dev); + + return ret; +} + +void udma_cleanup_common_hem(struct udma_dev *udma_dev) +{ + if (udma_dev->caps.gmv_entry_sz) + udma_cleanup_hem_table(udma_dev, &udma_dev->gmv_table); + if (udma_dev->caps.flags & UDMA_CAP_FLAG_QP_FLOW_CTRL) + udma_cleanup_hem_table(udma_dev, + &udma_dev->qp_table.sccc_table); + if (udma_dev->caps.flags & UDMA_CAP_FLAG_SRQ) + udma_cleanup_hem_table(udma_dev, &udma_dev->jfr_table.table); + udma_cleanup_hem_table(udma_dev, &udma_dev->jfc_table.table); + if (udma_dev->caps.trrl_entry_sz) + udma_cleanup_hem_table(udma_dev, + &udma_dev->qp_table.trrl_table); + + udma_cleanup_hem_table(udma_dev, &udma_dev->qp_table.irrl_table); + udma_cleanup_hem_table(udma_dev, &udma_dev->qp_table.qp_table); + udma_cleanup_hem_table(udma_dev, &udma_dev->seg_table.table); +} + +static void udma_cleanup_hem(struct udma_dev *udma_dev) +{ + if (udma_dev->caps.qpc_timer_entry_sz) + udma_cleanup_hem_table(udma_dev, &udma_dev->qpc_timer_table); + if (udma_dev->caps.cqc_timer_entry_sz) + udma_cleanup_hem_table(udma_dev, &udma_dev->cqc_timer_table); + + udma_cleanup_common_hem(udma_dev); +} + +void udma_set_poe_ch_num(struct udma_dev *dev) +{ +#define UDMA_POE_CH_NUM 4 + + dev->caps.poe_ch_num = UDMA_POE_CH_NUM; +} + +static void udma_set_devname(struct udma_dev *udma_dev, + struct ubcore_device *ub_dev) +{ + scnprintf(udma_dev->dev_name, UBCORE_MAX_DEV_NAME, "udma%d", + udma_dev->func_id); + dev_info(udma_dev->dev, "Set dev_name %s\n", udma_dev->dev_name); + strlcpy(ub_dev->dev_name, udma_dev->dev_name, UBCORE_MAX_DEV_NAME); +} + +static int udma_register_device(struct udma_dev *udma_dev) +{ + struct ubcore_device *ub_dev = NULL; + struct udma_netdev *uboe = NULL; + + ub_dev = &udma_dev->ub_dev; + uboe = &udma_dev->uboe; + spin_lock_init(&uboe->lock); + ub_dev->transport_type = UBCORE_TRANSPORT_IB; + ub_dev->ops = &g_udma_dev_ops; + ub_dev->dev.parent = udma_dev->dev; + ub_dev->dma_dev = ub_dev->dev.parent; + ub_dev->netdev = udma_dev->uboe.netdevs[0]; + scnprintf(ub_dev->ops->driver_name, UBCORE_MAX_DRIVER_NAME, "udma_v1"); + udma_set_devname(udma_dev, ub_dev); + ub_dev->num_comp_vectors = udma_dev->irq_num; + + return ubcore_register_device(ub_dev); +} + +static void udma_unregister_device(struct udma_dev *udma_dev) +{ + struct ubcore_device *ub_dev = &udma_dev->ub_dev; + + ubcore_unregister_device(ub_dev); +} + +int udma_hnae_client_init(struct udma_dev *udma_dev) +{ + struct device *dev = udma_dev->dev; + int ret; + + udma_dev->is_reset = false; + + ret = udma_dev->hw->cmq_init(udma_dev); + if (ret) { + dev_err(dev, "Init UB Command Queue failed!\n"); + goto error_failed_cmq_init; + } + + ret = udma_dev->hw->hw_profile(udma_dev); + if (ret) { + dev_err(dev, "Get UB engine profile failed!\n"); + goto error_failed_hw_profile; + } + + ret = udma_cmd_init(udma_dev); + if (ret) { + dev_err(dev, "cmd init failed!\n"); + goto error_failed_cmd_init; + } + + ret = udma_dev->hw->init_eq(udma_dev); + if (ret) { + dev_err(dev, "eq init failed!\n"); + goto error_failed_eq_table; + } + + if (udma_dev->cmd_mod) { + ret = udma_cmd_use_events(udma_dev); + if (ret) { + udma_dev->cmd_mod = 0; + dev_warn(dev, + "Cmd event mode failed, set back to poll!\n"); + } + } + + ret = udma_init_hem(udma_dev); + if (ret) { + dev_err(dev, "init HEM(Hardware Entry Memory) failed!\n"); + goto error_failed_hem_init; + } + + ret = udma_setup_hca(udma_dev); + if (ret) { + dev_err(dev, "setup hca failed!\n"); + goto error_failed_setup; + } + ret = udma_dev->hw->hw_init(udma_dev); + if (ret) { + dev_err(dev, "hw_init failed!\n"); + goto error_failed_engine_init; + } + + udma_set_poe_ch_num(udma_dev); + ret = udma_register_device(udma_dev); + if (ret) { + dev_err(dev, "udma register device failed!\n"); + goto error_failed_register_device; + } + + return 0; + +error_failed_register_device: + udma_dev->hw->hw_exit(udma_dev); + +error_failed_engine_init: + udma_teardown_hca(udma_dev); + +error_failed_setup: + udma_cleanup_hem(udma_dev); + +error_failed_hem_init: + if (udma_dev->cmd_mod) + udma_cmd_use_polling(udma_dev); + + udma_dev->hw->cleanup_eq(udma_dev); + +error_failed_eq_table: + udma_cmd_cleanup(udma_dev); + +error_failed_cmd_init: +error_failed_hw_profile: + udma_dev->hw->cmq_exit(udma_dev); + +error_failed_cmq_init: + return ret; +} + +void udma_hnae_client_exit(struct udma_dev *udma_dev) +{ + udma_unregister_device(udma_dev); + + if (udma_dev->hw->hw_exit) + udma_dev->hw->hw_exit(udma_dev); + + udma_teardown_hca(udma_dev); + + udma_cleanup_hem(udma_dev); + + if (udma_dev->cmd_mod) + udma_cmd_use_polling(udma_dev); + + udma_dev->hw->cleanup_eq(udma_dev); + + udma_cmd_cleanup(udma_dev); + if (udma_dev->hw->cmq_exit) + udma_dev->hw->cmq_exit(udma_dev); +} diff --git a/drivers/ub/hw/hns3/hns3_udma_qp.c b/drivers/ub/hw/hns3/hns3_udma_qp.c new file mode 100644 index 0000000000000000000000000000000000000000..7b98f7188d832b53c3ecab88a2fd588c7d8a43f5 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_qp.c @@ -0,0 +1,2088 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include +#include +#include +#include "hns3_udma_abi.h" +#include "hns3_udma_dca.h" +#include "hns3_udma_jfs.h" +#include "hns3_udma_jfr.h" +#include "hns3_udma_jfc.h" +#include "hns3_udma_hem.h" +#include "hns3_udma_cmd.h" +#include "hns3_udma_jetty.h" +#include "hns3_udma_tp.h" +#include "hns3_udma_db.h" +#include "hns3_udma_qp.h" + +static bool um_spray_en; +static ushort um_data_udp_start; +static ushort um_udp_range; + +static void set_qpc_wqe_cnt(struct udma_qp *qp, + struct udma_qp_context *context, + struct udma_qp_context *context_mask) +{ + udma_reg_write(context, QPC_SGE_SHIFT, + to_udma_hem_entries_shift(qp->sge.sge_cnt, + qp->sge.sge_shift)); + udma_reg_clear(context_mask, QPC_SGE_SHIFT); + + udma_reg_write(context, QPC_SQ_SHIFT, ilog2(qp->sq.wqe_cnt)); + udma_reg_clear(context_mask, QPC_SQ_SHIFT); +} + +static void config_qp_sq_buf_mask(struct udma_qp_context *context_mask) +{ + udma_reg_clear(context_mask, QPC_SQ_CUR_BLK_ADDR_L); + udma_reg_clear(context_mask, QPC_SQ_CUR_BLK_ADDR_H); + udma_reg_clear(context_mask, QPC_SQ_CUR_SGE_BLK_ADDR_L); + udma_reg_clear(context_mask, QPC_SQ_CUR_SGE_BLK_ADDR_H); + udma_reg_clear(context_mask, QPC_RX_SQ_CUR_BLK_ADDR_L); + udma_reg_clear(context_mask, QPC_RX_SQ_CUR_BLK_ADDR_H); + udma_reg_clear(context_mask, QPC_WQE_SGE_BA_L); + udma_reg_clear(context_mask, QPC_WQE_SGE_BA_H); + udma_reg_clear(context_mask, QPC_SQ_HOP_NUM); + udma_reg_clear(context_mask, QPC_SGE_HOP_NUM); + udma_reg_clear(context_mask, QPC_WQE_SGE_BA_PG_SZ); + udma_reg_clear(context_mask, QPC_WQE_SGE_BUF_PG_SZ); +} + +static int config_qp_sq_buf(struct udma_dev *udma_device, + struct udma_qp *qp, + struct udma_qp_context *context, + struct udma_qp_context *context_mask) +{ + uint64_t sge_cur_blk = 0; + uint64_t sq_cur_blk = 0; + uint64_t wqe_sge_ba = 0; + int count; + + /* search qp buf's mtts */ + count = udma_mtr_find(udma_device, &qp->mtr, qp->sq.wqe_offset, + &sq_cur_blk, 1, &wqe_sge_ba); + if (count < 1) { + dev_err(udma_device->dev, "failed to find QP(0x%llx) SQ buf.\n", + qp->qpn); + return -EINVAL; + } + + context->wqe_sge_ba = cpu_to_le32(wqe_sge_ba >> WQE_SGE_BA_OFFSET); + + if (qp->sge.sge_cnt > 0) { + count = udma_mtr_find(udma_device, &qp->mtr, + qp->sge.wqe_offset, &sge_cur_blk, + 1, NULL); + if (count < 1) { + dev_err(udma_device->dev, + "failed to find QP(0x%llx) SGE buf.\n", qp->qpn); + return -EINVAL; + } + } + + /* + * In v2 engine, software pass context and context mask to hardware + * when modifying qp. If software need modify some fields in context, + * we should set all bits of the relevant fields in context mask to + * 0 at the same time, else set them to 0x1. + */ + udma_reg_write(context, QPC_SQ_CUR_BLK_ADDR_L, + lower_32_bits(to_udma_hw_page_addr(sq_cur_blk))); + udma_reg_write(context, QPC_SQ_CUR_BLK_ADDR_H, + upper_32_bits(to_udma_hw_page_addr(sq_cur_blk))); + udma_reg_write(context, QPC_SQ_CUR_SGE_BLK_ADDR_L, + lower_32_bits(to_udma_hw_page_addr(sge_cur_blk))); + udma_reg_write(context, QPC_SQ_CUR_SGE_BLK_ADDR_H, + upper_32_bits(to_udma_hw_page_addr(sge_cur_blk))); + udma_reg_write(context, QPC_RX_SQ_CUR_BLK_ADDR_L, + lower_32_bits(to_udma_hw_page_addr(sq_cur_blk))); + udma_reg_write(context, QPC_RX_SQ_CUR_BLK_ADDR_H, + upper_32_bits(to_udma_hw_page_addr(sq_cur_blk))); + udma_reg_write(context, QPC_WQE_SGE_BA_H, wqe_sge_ba >> + (WQE_SGE_BA_OFFSET + H_ADDR_OFFSET)); + udma_reg_write(context, QPC_SQ_HOP_NUM, + to_udma_hem_hopnum(udma_device->caps.wqe_sq_hop_num, + qp->sq.wqe_cnt)); + udma_reg_write(context, QPC_SGE_HOP_NUM, + to_udma_hem_hopnum(udma_device->caps.wqe_sge_hop_num, + qp->sge.sge_cnt)); + udma_reg_write(context, QPC_WQE_SGE_BA_PG_SZ, + to_udma_hw_page_shift(qp->mtr.hem_cfg.ba_pg_shift)); + udma_reg_write(context, QPC_WQE_SGE_BUF_PG_SZ, + to_udma_hw_page_shift(qp->mtr.hem_cfg.buf_pg_shift)); + + config_qp_sq_buf_mask(context_mask); + + return 0; +} + +static void udma_set_path(const struct udma_modify_tp_attr *attr, + struct udma_qp_context *context, + struct udma_qp_context *context_mask) +{ + if (attr == NULL) + return; + + udma_reg_write(context, QPC_HOPLIMIT, attr->hop_limit); + udma_reg_clear(context_mask, QPC_HOPLIMIT); + + udma_reg_write(context, QPC_GMV_IDX, attr->sgid_index); + udma_reg_clear(context_mask, QPC_GMV_IDX); + + memcpy(context->dgid, attr->dgid, sizeof(attr->dgid)); + memset(context_mask->dgid, 0, sizeof(attr->dgid)); + + udma_reg_write(&(context->ext), QPCEX_DEID_H, + *(uint32_t *)(&attr->dgid[SGID_H_SHIFT])); + udma_reg_clear(&context_mask->ext, QPCEX_DEID_H); + + udma_reg_write(context, QPC_SL, attr->priority); + udma_reg_clear(context_mask, QPC_SL); +} + +static int udma_pass_qpc_to_hw(struct udma_dev *udma_device, + struct udma_qp_context *context, + struct udma_qp_context *qpc_mask, + struct udma_qp *qp) +{ + struct udma_cmd_mailbox *mailbox; + struct udma_cmq_desc desc; + int qpc_size; + int ret; + + struct udma_mbox *mb = (struct udma_mbox *)desc.data; + + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mailbox = udma_alloc_cmd_mailbox(udma_device); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + mbox_desc_init(mb, mailbox->dma, 0, qp->qpn, UDMA_CMD_MODIFY_QPC); + qpc_size = udma_device->caps.qpc_sz; + memcpy(mailbox->buf, context, qpc_size); + memcpy(mailbox->buf + qpc_size, qpc_mask, qpc_size); + + ret = udma_cmd_mbox(udma_device, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); + + udma_free_cmd_mailbox(udma_device, mailbox); + + return ret; +} + +int udma_set_dca_buf(struct udma_dev *dev, struct udma_qp *qp) +{ + struct udma_qp_context ctx[2] = {}; + struct udma_qp_context *msk = ctx + 1; + struct udma_qp_context *qpc = ctx; + int ret; + + memset(msk, 0xff, dev->caps.qpc_sz); + + ret = config_qp_sq_buf(dev, qp, qpc, msk); + if (ret) { + dev_err(dev->dev, "failed to config sq qpc, ret = %d.\n", ret); + return ret; + } + + ret = udma_pass_qpc_to_hw(dev, qpc, msk, qp); + if (ret) { + dev_err(dev->dev, "failed to modify DCA buf, ret = %d.\n", ret); + return ret; + } + + return 0; +} + +static bool check_qp_timeout_cfg_range(struct udma_dev *udma_device, + const uint8_t *timeout) +{ + if (*timeout > QP_TIMEOUT_MAX) { + dev_warn(udma_device->dev, + "Local ACK timeout shall be 0 to 31.\n"); + return false; + } + + return true; +} + +static enum udma_mtu to_udma_mtu(enum ubcore_mtu core_mtu) +{ + switch (core_mtu) { + case UBCORE_MTU_256: + return UDMA_MTU_256; + case UBCORE_MTU_512: + return UDMA_MTU_512; + case UBCORE_MTU_1024: + return UDMA_MTU_1024; + case UBCORE_MTU_2048: + return UDMA_MTU_2048; + default: + return UDMA_MTU_4096; + } +} + +static inline enum ubcore_mtu get_mtu(struct udma_qp *qp, + const struct ubcore_tp_attr *attr) +{ + if (qp->qp_type == QPT_UD || attr == NULL) + return UBCORE_MTU_4096; + + return attr->mtu; +} + +static inline int udma_mtu_enum_to_int(enum ubcore_mtu mtu) +{ + switch (mtu) { + case UBCORE_MTU_256: return 256; + case UBCORE_MTU_512: return 512; + case UBCORE_MTU_1024: return 1024; + case UBCORE_MTU_2048: return 2048; + case UBCORE_MTU_4096: return 4096; + default: return -1; + } +} + +static int udma_alloc_reorder_cq_buf(struct udma_dev *udma_dev, + struct udma_qp_attr *qp_attr) +{ + struct udma_caps *caps = &udma_dev->caps; + int buff_sz; + + buff_sz = (1 << caps->reorder_cq_shift) * caps->cqe_sz; + qp_attr->reorder_cq_size = buff_sz; + qp_attr->reorder_cq_page = dma_alloc_coherent(udma_dev->dev, buff_sz, + &qp_attr->reorder_cq_addr, + GFP_KERNEL); + if (!qp_attr->reorder_cq_page) { + dev_err(udma_dev->dev, "Dma alloc coherent failed\n"); + return -ENOMEM; + } + + return 0; +} + +static void udma_free_reorder_cq_buf(struct udma_dev *udma_dev, + struct udma_qp_attr *qp_attr) +{ + if (qp_attr->reorder_cq_page) + dma_free_coherent(udma_dev->dev, qp_attr->reorder_cq_size, + qp_attr->reorder_cq_page, + qp_attr->reorder_cq_addr); +} + +static void edit_qpc_for_inline(struct udma_qp_context *context, + struct udma_qp_context *context_mask, + struct udma_qp *qp) +{ + struct udma_dev *udma_dev = qp->udma_device; + + if (qp->qp_type != QPT_UD) { + udma_reg_write(context, QPC_CQEIE, + !!(udma_dev->caps.flags & + UDMA_CAP_FLAG_CQE_INLINE)); + udma_reg_clear(context_mask, QPC_CQEIE); + } +} + +static void edit_qpc_for_db(struct udma_qp_context *context, struct udma_qp_context *context_mask, + struct udma_qp *qp) +{ + if (qp->en_flags & UDMA_QP_CAP_RQ_RECORD_DB) { + udma_reg_enable(context, QPC_RQ_RECORD_EN); + udma_reg_clear(context_mask, QPC_RQ_RECORD_EN); + } + + if (qp->en_flags & UDMA_QP_CAP_OWNER_DB) { + udma_reg_enable(context, QPC_OWNER_MODE); + udma_reg_clear(context_mask, QPC_OWNER_MODE); + } +} + +static void edit_qpc_for_srqn(struct udma_qp *qp, + struct udma_qp_context *context, + struct udma_qp_context *context_mask) +{ + if (qp->qp_attr.jfr) { + udma_reg_enable(context, QPC_SRQ_EN); + udma_reg_clear(context_mask, QPC_SRQ_EN); + + udma_reg_write(context, QPC_SRQN, qp->qp_attr.jfr->jfrn); + udma_reg_clear(context_mask, QPC_SRQN); + } +} + +static void edit_qpc_for_rxcqn(struct udma_qp *qp, + struct udma_qp_context *context, + struct udma_qp_context *context_mask) +{ + if (qp->recv_jfc) { + udma_reg_write(context, QPC_RX_CQN, qp->recv_jfc->cqn); + udma_reg_clear(context_mask, QPC_RX_CQN); + } +} + +static void edit_qpc_for_retransmission_parm(struct udma_dev *udma_device, + struct udma_qp *qp, + const struct udma_modify_tp_attr *attr, + struct udma_qp_context *context, + struct udma_qp_context *context_mask) +{ + if (qp->qp_type != QPT_UD) { + udma_reg_write(context, QPC_MIN_RNR_TIME, + attr->min_rnr_timer); + udma_reg_clear(context_mask, QPC_MIN_RNR_TIME); + + udma_reg_write(context, QPC_RETRY_CNT, + attr->retry_cnt); + udma_reg_clear(context_mask, QPC_RETRY_CNT); + + udma_reg_write(context, QPC_RETRY_NUM_INIT, + attr->retry_cnt); + udma_reg_clear(context_mask, QPC_RETRY_NUM_INIT); + + udma_reg_write(context, QPC_RNR_CNT, + attr->rnr_retry); + udma_reg_clear(context_mask, QPC_RNR_CNT); + + udma_reg_write(context, QPC_RNR_NUM_INIT, + attr->rnr_retry); + udma_reg_clear(context_mask, QPC_RNR_NUM_INIT); + + if (check_qp_timeout_cfg_range(udma_device, &attr->ack_timeout)) { + udma_reg_write(context, QPC_AT, attr->ack_timeout); + udma_reg_clear(context_mask, QPC_AT); + } + } +} + +static void edit_qpc_for_write(struct udma_qp *qp, + struct udma_qp_context *context, + struct udma_qp_context *context_mask) +{ + udma_reg_enable(context, QPC_FLUSH_EN); + udma_reg_clear(context_mask, QPC_FLUSH_EN); + + udma_reg_enable(context, QPC_AW_EN); + udma_reg_clear(context_mask, QPC_AW_EN); + + udma_reg_enable(context, QPC_WN_EN); + udma_reg_clear(context_mask, QPC_WN_EN); + + udma_reg_enable(context, QPC_RMT_E2E); + udma_reg_clear(context_mask, QPC_RMT_E2E); + + udma_reg_write(context, QPC_SIG_TYPE, SIGNAL_REQ_WR); + udma_reg_clear(context_mask, QPC_SIG_TYPE); +} + +static void edit_qpc_for_receive(struct udma_qp *qp, + const struct udma_modify_tp_attr *attr, + struct udma_qp_context *context, + struct udma_qp_context *context_mask) +{ + uint8_t *dmac; + + dmac = (uint8_t *)attr->dmac; + memcpy(&context->dmac, dmac, sizeof(uint32_t)); + udma_reg_write(context, QPC_DMAC_L, *((uint32_t *)(&dmac[0]))); + udma_reg_clear(context_mask, QPC_DMAC_L); + + udma_reg_write(context, QPC_DMAC_H, + *((uint16_t *)(&dmac[QPC_DMAC_H_IDX]))); + udma_reg_clear(context_mask, QPC_DMAC_H); + + context->rq_rnr_timer = 0; + context_mask->rq_rnr_timer = 0; + + /* rocee send 2^lp_sgen_ini segs every time */ + udma_reg_write(context, QPC_LP_SGEN_INI, SGEN_INI_VALUE); + udma_reg_clear(context_mask, QPC_LP_SGEN_INI); +} + +static int modify_qp_reset_to_rtr(struct udma_qp *qp, + const struct udma_modify_tp_attr *attr, + struct udma_qp_context *context, + struct udma_qp_context *context_mask) +{ + struct udma_dev *udma_device = qp->udma_device; + + udma_reg_write(context, QPC_RRE, 1); + udma_reg_clear(context_mask, QPC_RRE); + + udma_reg_write(context, QPC_RWE, 1); + udma_reg_clear(context_mask, QPC_RWE); + + udma_reg_write(context, QPC_TST, qp->qp_type); + udma_reg_clear(context_mask, QPC_TST); + + udma_reg_write(context, QPC_RQWS, ilog2(qp->rq.max_gs)); + udma_reg_clear(context_mask, QPC_RQWS); + + /* No VLAN need to set 0xFFF */ + udma_reg_write(context, QPC_VLAN_ID, 0xfff); + udma_reg_clear(context_mask, QPC_VLAN_ID); + + edit_qpc_for_db(context, context_mask, qp); + + edit_qpc_for_inline(context, context_mask, qp); + + udma_reg_write(&context->ext, QPCEX_P_TYPE, QPCEX_P_TYPE_UDMA); + udma_reg_clear(&context_mask->ext, QPCEX_P_TYPE); + + edit_qpc_for_srqn(qp, context, context_mask); + + edit_qpc_for_retransmission_parm(udma_device, qp, attr, context, context_mask); + + edit_qpc_for_rxcqn(qp, context, context_mask); + + /* + * Enable atomic WRITE and persistence WRITE and Write With Notify + * operations in QPC when modify_qp_init_to_rtr. + */ + edit_qpc_for_write(qp, context, context_mask); + + edit_qpc_for_receive(qp, attr, context, context_mask); + return 0; +} + +static int modify_qp_rtr_to_rts(struct udma_qp *qp, + struct udma_qp_context *context, + struct udma_qp_context *context_mask) +{ + struct udma_dev *udma_device = qp->udma_device; + int ret; + + qp->sq.wqe_offset = qp->sq.offset; + qp->sge.wqe_offset = qp->sge.offset; + + ret = config_qp_sq_buf(udma_device, qp, context, context_mask); + if (ret) { + dev_err(udma_device->dev, "failed to config sq buf, ret = %d.\n", + ret); + return ret; + } + if (qp->send_jfc) { + udma_reg_write(context, QPC_TX_CQN, qp->send_jfc->cqn); + udma_reg_clear(context_mask, QPC_TX_CQN); + } + + if (qp->en_flags & UDMA_QP_CAP_DYNAMIC_CTX_ATTACH) { + udma_reg_enable(context, QPC_DCA_MODE); + udma_reg_clear(context_mask, QPC_DCA_MODE); + } + + set_qpc_wqe_cnt(qp, context, context_mask); + + return 0; +} + +static enum udma_cong_type to_udma_cong_type(uint32_t cc_alg) +{ + switch ((enum ubcore_tp_cc_alg)cc_alg) { + case UBCORE_TP_CC_DCQCN: + return UDMA_CONG_TYPE_DCQCN; + case UBCORE_TP_CC_LDCP: + return UDMA_CONG_TYPE_LDCP; + case UBCORE_TP_CC_HC3: + return UDMA_CONG_TYPE_HC3; + case UBCORE_TP_CC_DIP: + return UDMA_CONG_TYPE_DIP; + default: + return UDMA_CONG_TYPE_DCQCN; + } +} + +static void fill_congest_type(struct udma_congestion_algorithm *congest_alg, + enum udma_cong_type qp_cong_alg) +{ + switch (qp_cong_alg) { + case UDMA_CONG_TYPE_DCQCN: + congest_alg->congest_type = CONGEST_DCQCN; + congest_alg->alg_sel = DCQCN_ALG; + congest_alg->alg_sub_sel = UNSUPPORT_CONGEST_DEGREE; + congest_alg->dip_vld = DIP_INVALID; + congest_alg->wnd_mode_sel = WND_LIMIT; + break; + case UDMA_CONG_TYPE_LDCP: + congest_alg->congest_type = CONGEST_LDCP; + congest_alg->alg_sel = WINDOW_ALG; + congest_alg->alg_sub_sel = UNSUPPORT_CONGEST_DEGREE; + congest_alg->dip_vld = DIP_INVALID; + congest_alg->wnd_mode_sel = WND_UNLIMIT; + break; + case UDMA_CONG_TYPE_HC3: + congest_alg->congest_type = CONGEST_HC3; + congest_alg->alg_sel = WINDOW_ALG; + congest_alg->alg_sub_sel = SUPPORT_CONGEST_DEGREE; + congest_alg->dip_vld = DIP_INVALID; + congest_alg->wnd_mode_sel = WND_LIMIT; + break; + case UDMA_CONG_TYPE_DIP: + congest_alg->congest_type = CONGEST_DIP; + congest_alg->alg_sel = DCQCN_ALG; + congest_alg->alg_sub_sel = UNSUPPORT_CONGEST_DEGREE; + congest_alg->dip_vld = DIP_VALID; + congest_alg->wnd_mode_sel = WND_LIMIT; + break; + default: + congest_alg->congest_type = CONGEST_DCQCN; + congest_alg->alg_sel = DCQCN_ALG; + congest_alg->alg_sub_sel = UNSUPPORT_CONGEST_DEGREE; + congest_alg->dip_vld = DIP_INVALID; + congest_alg->wnd_mode_sel = WND_LIMIT; + break; + } +} + +static int get_dip_ctx_idx(struct udma_qp *qp, + uint32_t *dip_idx) +{ + struct udma_dev *udma_dev = qp->udma_device; + uint32_t *spare_idx = udma_dev->qp_table.idx_table.spare_idx; + uint32_t *head = &udma_dev->qp_table.idx_table.head; + uint32_t *tail = &udma_dev->qp_table.idx_table.tail; + struct udma_modify_tp_attr *attr; + struct udma_dip *udma_dip; + unsigned long flags; + int ret = 0; + + attr = qp->m_attr; + spin_lock_irqsave(&udma_dev->dip_list_lock, flags); + + spare_idx[*tail] = qp->qpn; + *tail = (*tail == udma_dev->caps.num_qps - 1) ? 0 : (*tail + 1); + + list_for_each_entry(udma_dip, &udma_dev->dip_list, node) { + if (!memcmp(attr->dgid, udma_dip->dgid, UDMA_GID_SIZE)) { + *dip_idx = udma_dip->dip_idx; + goto out; + } + } + + udma_dip = kzalloc(sizeof(*udma_dip), GFP_ATOMIC); + if (!udma_dip) { + ret = -ENOMEM; + goto out; + } + + memcpy(udma_dip->dgid, attr->dgid, sizeof(attr->dgid)); + udma_dip->dip_idx = *dip_idx = spare_idx[*head]; + *head = (*head == udma_dev->caps.num_qps - 1) ? 0 : (*head + 1); + list_add_tail(&udma_dip->node, &udma_dev->dip_list); + +out: + spin_unlock_irqrestore(&udma_dev->dip_list_lock, flags); + return ret; +} + +static int udma_set_cong_fields(struct udma_qp_context *context, + struct udma_qp_context *context_mask, + struct udma_qp *qp, + const struct ubcore_tp_attr *attr) +{ + struct udma_congestion_algorithm congest_filed; + enum udma_cong_type qp_cong_alg; + uint32_t dip_idx = 0; + int ret = 0; + + qp_cong_alg = to_udma_cong_type(attr->flag.bs.cc_alg); + fill_congest_type(&congest_filed, qp_cong_alg); + + if (qp->qp_type == QPT_UD) { + congest_filed.congest_type = CONGEST_DCQCN; + congest_filed.alg_sel = DCQCN_ALG; + congest_filed.alg_sub_sel = UNSUPPORT_CONGEST_DEGREE; + congest_filed.dip_vld = DIP_INVALID; + congest_filed.wnd_mode_sel = WND_LIMIT; + } + + udma_reg_write(context, QPC_CONGEST_ALGO_TMPL_ID, + qp->udma_device->cong_algo_tmpl_id + + congest_filed.congest_type * UDMA_CONGEST_SIZE); + udma_reg_clear(context_mask, QPC_CONGEST_ALGO_TMPL_ID); + + udma_reg_write(&context->ext, QPCEX_CONGEST_ALG_SEL, congest_filed.alg_sel); + udma_reg_clear(&context_mask->ext, QPCEX_CONGEST_ALG_SEL); + + udma_reg_write(&context->ext, QPCEX_CONGEST_ALG_SUB_SEL, congest_filed.alg_sub_sel); + udma_reg_clear(&context_mask->ext, QPCEX_CONGEST_ALG_SUB_SEL); + + udma_reg_write(&context->ext, QPCEX_DIP_CTX_IDX_VLD, congest_filed.dip_vld); + udma_reg_clear(&context_mask->ext, QPCEX_DIP_CTX_IDX_VLD); + + udma_reg_write(&context->ext, QPCEX_SQ_RQ_NOT_FORBID_EN, congest_filed.wnd_mode_sel); + udma_reg_clear(&context_mask->ext, QPCEX_SQ_RQ_NOT_FORBID_EN); + + if (congest_filed.dip_vld == 0) + goto out; + + ret = get_dip_ctx_idx(qp, &dip_idx); + if (ret) { + dev_err(qp->udma_device->dev, + "failed to fill congest, ret = %d.\n", ret); + goto out; + } + qp->dip_idx = (int64_t)dip_idx; + if (dip_idx != qp->qpn) { + ret = udma_table_get(qp->udma_device, + &qp->udma_device->qp_table.sccc_table, + dip_idx); + if (ret) { + dev_err(qp->udma_device->dev, + "Failed to get SCC CTX table\n"); + goto out; + } + } + + udma_reg_write(&context->ext, QPCEX_DIP_CTX_IDX, dip_idx); + udma_reg_clear(&context_mask->ext, QPCEX_DIP_CTX_IDX); + +out: + return ret; +} + +static void udma_set_spray_field(struct udma_qp *qp, + const struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask ubcore_mask, + struct udma_qp_context *context, + struct udma_qp_context *context_mask) +{ + struct udma_dev *udma_dev = qp->udma_device; + struct udma_modify_tp_attr *m_attr; + uint16_t dus_regval; + uint16_t aus_regval; + uint16_t real_range; + + m_attr = qp->m_attr; + real_range = (m_attr->udp_range + + UDP_SRCPORT_RANGE_BASE) & + UDP_SRCPORT_RANGE_SIZE_MASK; + dus_regval = m_attr->data_udp_start & + GENMASK(real_range, 0); + aus_regval = m_attr->ack_udp_start & + GENMASK(real_range, 0); + + udma_reg_enable(&context->ext, QPCEX_AR_EN); + udma_reg_clear(&context_mask->ext, QPCEX_AR_EN); + + udma_reg_write(&context->ext, QPCEX_ACK_UDP_SRCPORT, + aus_regval); + udma_reg_clear(&context_mask->ext, QPCEX_ACK_UDP_SRCPORT); + + udma_reg_write(&context->ext, QPCEX_DATA_UDP_SRCPORT_L, + dus_regval); + udma_reg_clear(&context_mask->ext, QPCEX_DATA_UDP_SRCPORT_L); + + udma_reg_write(&context->ext, QPCEX_DATA_UDP_SRCPORT_H, + dus_regval >> + QPCEX_DATA_UDP_SRCPORT_H_SHIFT); + udma_reg_clear(&context_mask->ext, QPCEX_DATA_UDP_SRCPORT_H); + + udma_reg_write(&context->ext, QPCEX_UDP_SRCPORT_RANGE, + m_attr->udp_range); + udma_reg_clear(&context_mask->ext, QPCEX_UDP_SRCPORT_RANGE); + + if (udma_dev->caps.reorder_cq_buffer_en && + qp->qp_attr.reorder_cq_addr) { + udma_reg_enable(&context->ext, QPCEX_REORDER_CQ_EN); + udma_reg_clear(&context_mask->ext, QPCEX_REORDER_CQ_EN); + + udma_reg_write(&context->ext, QPCEX_REORDER_CQ_ADDR_L, + lower_32_bits(qp->qp_attr.reorder_cq_addr) >> + QPCEX_REORDER_CQ_ADDR_SHIFT); + udma_reg_clear(&context_mask->ext, QPCEX_REORDER_CQ_ADDR_L); + + udma_reg_write(&context->ext, QPCEX_REORDER_CQ_ADDR_H, + upper_32_bits(qp->qp_attr.reorder_cq_addr)); + udma_reg_clear(&context_mask->ext, QPCEX_REORDER_CQ_ADDR_H); + + udma_reg_write(&context->ext, QPCEX_REORDER_CQ_SHIFT, + udma_dev->caps.reorder_cq_shift); + udma_reg_clear(&context_mask->ext, QPCEX_REORDER_CQ_SHIFT); + } +} + +static void udma_set_oor_field(struct udma_qp *qp, + const struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask ubcore_mask, + struct udma_qp_context *context, + struct udma_qp_context *context_mask) +{ + struct udma_dev *udma_dev = qp->udma_device; + + udma_reg_enable(&context->ext, QPCEX_OOR_EN); + udma_reg_clear(&context_mask->ext, QPCEX_OOR_EN); + + udma_reg_write(&context->ext, QPCEX_REORDER_CAP, + udma_dev->caps.reorder_cap); + udma_reg_clear(&context_mask->ext, QPCEX_REORDER_CAP); + + udma_reg_write(&context->ext, QPCEX_ON_FLIGHT_SIZE_L, + udma_dev->caps.onflight_size); + udma_reg_clear(&context_mask->ext, QPCEX_ON_FLIGHT_SIZE_L); + + udma_reg_write(&context->ext, QPCEX_ON_FLIGHT_SIZE_H, + udma_dev->caps.onflight_size >> + QPCEX_ON_FLIGHT_SIZE_H_SHIFT); + udma_reg_clear(&context_mask->ext, QPCEX_ON_FLIGHT_SIZE_H); + + udma_reg_write(&context->ext, QPCEX_DYN_AT, + udma_dev->caps.dynamic_ack_timeout); + udma_reg_clear(&context_mask->ext, QPCEX_DYN_AT); +} + +static void udma_set_opt_fields(struct udma_qp *qp, + const struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask ubcore_mask, + struct udma_qp_context *context, + struct udma_qp_context *context_mask) +{ + struct udma_dev *udma_dev = qp->udma_device; + uint8_t lp_pktn_ini; + + if (attr == NULL) + return; + + if (ubcore_mask.bs.flag && attr->flag.bs.oor_en && udma_dev->caps.oor_en) + udma_set_oor_field(qp, attr, ubcore_mask, context, context_mask); + + if (ubcore_mask.bs.flag) + udma_set_cong_fields(context, context_mask, qp, attr); + + if (ubcore_mask.bs.flag && attr->flag.bs.spray_en && + (udma_dev->caps.flags & UDMA_CAP_FLAG_AR)) + udma_set_spray_field(qp, attr, ubcore_mask, context, context_mask); + + if (ubcore_mask.bs.peer_tpn) { + udma_reg_write(context, QPC_DQPN, attr->peer_tpn); + udma_reg_clear(context_mask, QPC_DQPN); + } + + if (ubcore_mask.bs.tx_psn) { + udma_reg_write(context, QPC_RX_ACK_EPSN, attr->tx_psn); + udma_reg_write(context, QPC_RETRY_MSG_PSN_L, attr->tx_psn); + udma_reg_write(context, QPC_RETRY_MSG_PSN_H, + attr->tx_psn >> RETRY_MSG_PSN_H_OFFSET); + udma_reg_write(context, QPC_RETRY_MSG_FPKT_PSN, attr->tx_psn); + udma_reg_write(context, QPC_SQ_CUR_PSN, attr->tx_psn); + udma_reg_write(context, QPC_SQ_MAX_PSN, attr->tx_psn); + + udma_reg_clear(context_mask, QPC_RX_ACK_EPSN); + udma_reg_clear(context_mask, QPC_RETRY_MSG_PSN_L); + udma_reg_clear(context_mask, QPC_RETRY_MSG_PSN_H); + udma_reg_clear(context_mask, QPC_RETRY_MSG_FPKT_PSN); + udma_reg_clear(context_mask, QPC_SQ_CUR_PSN); + udma_reg_clear(context_mask, QPC_SQ_MAX_PSN); + } + + if (ubcore_mask.bs.rx_psn) { + udma_reg_write(context, QPC_RX_REQ_EPSN, attr->rx_psn); + udma_reg_write(context, QPC_RAQ_PSN, attr->rx_psn - 1); + + udma_reg_clear(context_mask, QPC_RX_REQ_EPSN); + udma_reg_clear(context_mask, QPC_RAQ_PSN); + } + + if (ubcore_mask.bs.mtu) { + qp->ubcore_path_mtu = get_mtu(qp, attr); + qp->path_mtu = to_udma_mtu(qp->ubcore_path_mtu); + udma_reg_write(context, QPC_MTU, qp->path_mtu); + udma_reg_clear(context_mask, QPC_MTU); + + /* MTU * (2 ^ LP_PKTN_INI) shouldn't be bigger than 16KB */ + lp_pktn_ini = ilog2(MAX_LP_MSG_LEN / udma_mtu_enum_to_int(qp->path_mtu)); + + udma_reg_write(context, QPC_LP_PKTN_INI, lp_pktn_ini); + udma_reg_clear(context_mask, QPC_LP_PKTN_INI); + + /* ACK_REQ_FREQ should be larger than or equal to LP_PKTN_INI */ + udma_reg_write(context, QPC_ACK_REQ_FREQ, lp_pktn_ini); + udma_reg_clear(context_mask, QPC_ACK_REQ_FREQ); + } +} + +static int udma_set_abs_fields(struct udma_qp *qp, + const struct udma_modify_tp_attr *attr, + enum udma_qp_state curr_state, + enum udma_qp_state new_state, + struct udma_qp_context *context, + struct udma_qp_context *context_mask) +{ + struct udma_dev *udma_device = qp->udma_device; + int ret = 0; + + if (curr_state == QPS_RESET && new_state == QPS_RTR) { + ret = modify_qp_reset_to_rtr(qp, attr, context, context_mask); + if (ret) { + dev_err(udma_device->dev, + "Something went wrong during modify_qp_init_to_rtr\n"); + goto out; + } + } else if (curr_state == QPS_RESET && new_state == QPS_RTS) { + ret = modify_qp_reset_to_rtr(qp, attr, context, context_mask); + if (ret) { + dev_err(udma_device->dev, + "Something went wrong during modify_qp_init_to_rtr\n"); + goto out; + } + ret = modify_qp_rtr_to_rts(qp, context, context_mask); + if (ret) { + dev_err(udma_device->dev, + "Something went wrong during modify_qp_rtr_to_rts\n"); + goto out; + } + } else if (curr_state == QPS_RTR && new_state == QPS_RTS) { + ret = modify_qp_rtr_to_rts(qp, context, context_mask); + if (ret) { + dev_err(udma_device->dev, + "Something went wrong during modify_qp_rtr_to_rts\n"); + goto out; + } + } + +out: + return ret; +} + +static void udma_set_um_attr(struct udma_qp *qp, + struct udma_qp_context *context, + struct udma_qp_context *context_mask) +{ + uint8_t lp_pktn_ini; + + qp->ubcore_path_mtu = get_mtu(qp, NULL); + qp->path_mtu = to_udma_mtu(qp->ubcore_path_mtu); + udma_reg_write(context, QPC_MTU, qp->path_mtu); + udma_reg_clear(context_mask, QPC_MTU); + + /* MTU * (2 ^ LP_PKTN_INI) shouldn't be bigger than 16KB */ + lp_pktn_ini = ilog2(MAX_LP_MSG_LEN / udma_mtu_enum_to_int(qp->path_mtu)); + + udma_reg_write(context, QPC_LP_PKTN_INI, lp_pktn_ini); + udma_reg_clear(context_mask, QPC_LP_PKTN_INI); + + /* ACK_REQ_FREQ should be larger than or equal to LP_PKTN_INI */ + udma_reg_write(context, QPC_ACK_REQ_FREQ, lp_pktn_ini); + udma_reg_clear(context_mask, QPC_ACK_REQ_FREQ); +} + +int udma_modify_qp_common(struct udma_qp *qp, + const struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask ubcore_mask, + enum udma_qp_state curr_state, + enum udma_qp_state new_state) +{ + struct udma_dev *udma_device = qp->udma_device; + struct udma_qp_context ctx[2] = {}; + struct udma_qp_context *context = ctx; + struct udma_qp_context *context_mask = ctx + 1; + int ret = 0; + + memset(context, 0, udma_device->caps.qpc_sz); + memset(context_mask, 0xff, udma_device->caps.qpc_sz); + if (new_state != QPS_RESET) { + ret = udma_set_abs_fields(qp, qp->m_attr, curr_state, new_state, + context, context_mask); + if (ret) + goto out; + } + + if (qp->qp_type == QPT_UD) + udma_set_um_attr(qp, context, context_mask); + + udma_set_opt_fields(qp, attr, ubcore_mask, context, context_mask); + + udma_reg_write(context, QPC_INV_CREDIT, (!!qp->qp_attr.jfr) ? 1 : 0); + udma_reg_clear(context_mask, QPC_INV_CREDIT); + /* Every status migrate must change state */ + udma_reg_write(context, QPC_QP_ST, new_state); + udma_reg_clear(context_mask, QPC_QP_ST); + + udma_set_path(qp->m_attr, context, context_mask); + + udma_reg_write(&context->ext, QPCEX_P_TYPE, QPCEX_P_TYPE_UDMA); + udma_reg_clear(&context_mask->ext, QPCEX_P_TYPE); + /* SW pass context to HW */ + ret = udma_pass_qpc_to_hw(udma_device, context, context_mask, qp); + if (ret) { + dev_err(udma_device->dev, "failed to pass QPC to HW, ret = %d.\n", + ret); + goto out; + } + + qp->state = new_state; + + if (qp->qp_type == QPT_RC && + qp->en_flags & UDMA_QP_CAP_DYNAMIC_CTX_ATTACH) + udma_modify_dca(udma_device, qp); + +out: + return ret; +} + +int fill_jfs_qp_attr(struct udma_dev *udma_dev, struct udma_qp_attr *qp_attr, + struct udma_create_tp_ucmd *ucmd) +{ + struct udma_jfs *udma_jfs; + struct ubcore_jfs *jfs; + struct ubcore_jfc *jfc; + + udma_jfs = (struct udma_jfs *)xa_load(&udma_dev->jfs_table.xa, + ucmd->ini_id.jfs_id); + if (IS_ERR_OR_NULL(udma_jfs)) { + dev_err(udma_dev->dev, "failed to find jfs\n"); + return -EINVAL; + } + jfs = &udma_jfs->ubcore_jfs; + jfc = jfs->jfs_cfg.jfc; + qp_attr->send_jfc = to_udma_jfc(jfc); + qp_attr->recv_jfc = NULL; + qp_attr->jfs = udma_jfs; + qp_attr->qpn_map = &qp_attr->jfs->qpn_map; + qp_attr->uctx = qp_attr->jfs->ubcore_jfs.uctx; + qp_attr->cap.max_send_wr = jfs->jfs_cfg.depth; + qp_attr->cap.max_send_sge = jfs->jfs_cfg.max_sge; + qp_attr->cap.max_inline_data = jfs->jfs_cfg.max_inline_data; + qp_attr->cap.retry_cnt = jfs->jfs_cfg.retry_cnt; + qp_attr->cap.rnr_retry = jfs->jfs_cfg.rnr_retry; + qp_attr->cap.ack_timeout = jfs->jfs_cfg.err_timeout; + qp_attr->qp_type = QPT_RC; + qp_attr->tgt_id = ucmd->tgt_id.jfr_id; + if (jfs->jfs_cfg.priority >= udma_dev->caps.sl_num) { + qp_attr->priority = udma_dev->caps.sl_num > 0 ? + udma_dev->caps.sl_num - 1 : 0; + dev_err(udma_dev->dev, + "The setted priority (%d) cannot larger than the max priority (%d), priority (%d) is used.\n", + jfs->jfs_cfg.priority, udma_dev->caps.sl_num, + qp_attr->priority); + } else { + qp_attr->priority = jfs->jfs_cfg.priority; + } + + return 0; +} + +int fill_jfr_qp_attr(struct udma_dev *udma_dev, struct udma_qp_attr *qp_attr, + struct udma_create_tp_ucmd *ucmd) +{ + struct udma_jfr *udma_jfr; + struct ubcore_jfr *jfr; + + udma_jfr = (struct udma_jfr *)xa_load(&udma_dev->jfr_table.xa, + ucmd->tgt_id.jfr_id); + if (IS_ERR_OR_NULL(udma_jfr)) { + dev_err(udma_dev->dev, "failed to find jfr\n"); + return -EINVAL; + } + jfr = &udma_jfr->ubcore_jfr; + qp_attr->jfr = udma_jfr; + qp_attr->recv_jfc = to_udma_jfc(jfr->jfr_cfg.jfc); + qp_attr->uctx = qp_attr->jfr->ubcore_jfr.uctx; + qp_attr->qpn_map = &qp_attr->jfr->qpn_map; + + if (jfr->jfr_cfg.trans_mode == UBCORE_TP_UM) { + dev_err(udma_dev->dev, "jfr tp mode error\n"); + return -EINVAL; + } else { + qp_attr->qp_type = QPT_RC; + } + qp_attr->cap.min_rnr_timer = jfr->jfr_cfg.min_rnr_timer; + + return 0; +} + +int fill_jetty_qp_attr(struct udma_dev *udma_dev, struct udma_qp_attr *qp_attr, + struct udma_create_tp_ucmd *ucmd) +{ + struct udma_jetty *udma_jetty; + struct ubcore_jetty *jetty; + uint32_t jetty_id; + + jetty_id = qp_attr->is_tgt ? ucmd->tgt_id.jetty_id : + ucmd->ini_id.jetty_id; + qp_attr->tgt_id = qp_attr->is_tgt ? ucmd->ini_id.jetty_id : + ucmd->tgt_id.jetty_id; + + udma_jetty = (struct udma_jetty *)xa_load(&udma_dev->jetty_table.xa, + jetty_id); + if (IS_ERR_OR_NULL(udma_jetty)) { + dev_err(udma_dev->dev, "failed to find jetty\n"); + return -EINVAL; + } + + jetty = &udma_jetty->ubcore_jetty; + if (udma_jetty->tp_mode == UBCORE_TP_UM) { + dev_err(udma_dev->dev, "jetty tp mode error\n"); + return -EINVAL; + } + + qp_attr->jetty = udma_jetty; + if (!qp_attr->is_tgt || udma_jetty->tp_mode == UBCORE_TP_RC) { + qp_attr->uctx = jetty->uctx; + qp_attr->qpn_map = &udma_jetty->qpn_map; + qp_attr->send_jfc = to_udma_jfc(jetty->jetty_cfg.send_jfc); + qp_attr->cap.max_send_wr = jetty->jetty_cfg.jfs_depth; + qp_attr->cap.max_send_sge = jetty->jetty_cfg.max_send_sge; + qp_attr->cap.max_inline_data = jetty->jetty_cfg.max_inline_data; + if (jetty->jetty_cfg.priority >= udma_dev->caps.sl_num) { + qp_attr->priority = udma_dev->caps.sl_num > 0 ? + udma_dev->caps.sl_num - 1 : 0; + dev_err(udma_dev->dev, + "The setted priority (%d) should smaller than the max priority (%d), priority (%d) is used\n", + jetty->jetty_cfg.priority, + udma_dev->caps.sl_num, qp_attr->priority); + } else { + qp_attr->priority = jetty->jetty_cfg.priority; + } + } + + qp_attr->jfr = udma_jetty->udma_jfr; + qp_attr->uctx = udma_jetty->udma_jfr->ubcore_jfr.uctx; + qp_attr->qpn_map = &udma_jetty->qpn_map; + qp_attr->recv_jfc = + to_udma_jfc(udma_jetty->udma_jfr->ubcore_jfr.jfr_cfg.jfc); + + qp_attr->qp_type = QPT_RC; + qp_attr->cap.min_rnr_timer = + udma_jetty->udma_jfr->ubcore_jfr.jfr_cfg.min_rnr_timer; + + qp_attr->cap.retry_cnt = jetty->jetty_cfg.retry_cnt; + qp_attr->cap.ack_timeout = jetty->jetty_cfg.err_timeout; + qp_attr->cap.rnr_retry = jetty->jetty_cfg.rnr_retry; + + return 0; +} + +int udma_fill_qp_attr(struct udma_dev *udma_dev, struct udma_qp_attr *qp_attr, + const struct ubcore_tp_cfg *cfg, struct ubcore_udata *udata) +{ + bool is_target = udata->uctx == NULL ? true : false; + struct udma_create_tp_ucmd ucmd; + struct udma_ucontext *udma_ctx; + int status = 0; + + if (!udata) + return 0; + + if (!is_target) { + status = copy_from_user(&ucmd, (void *)udata->udrv_data->in_addr, + min(udata->udrv_data->in_len, + (uint32_t)sizeof(ucmd))); + if (status) { + dev_err(udma_dev->dev, "failed to copy create tp ucmd\n"); + return status; + } + } else { + memcpy(&ucmd, (void *)udata->udrv_data->in_addr, + min(udata->udrv_data->in_len, (uint32_t)sizeof(ucmd))); + } + + qp_attr->is_tgt = is_target; + qp_attr->is_jetty = ucmd.is_jetty; + qp_attr->remote_eid = cfg->peer_eid; + qp_attr->local_eid = cfg->local_eid; + udma_ctx = to_udma_ucontext(udata->uctx); + + if (!is_target) { + qp_attr->pdn = udma_ctx->pdn; + if (!ucmd.is_jetty) + return fill_jfs_qp_attr(udma_dev, qp_attr, &ucmd); + else + return fill_jetty_qp_attr(udma_dev, qp_attr, &ucmd); + } else { + if (!ucmd.is_jetty) + return fill_jfr_qp_attr(udma_dev, qp_attr, &ucmd); + else + return fill_jetty_qp_attr(udma_dev, qp_attr, &ucmd); + } + + return status; +} + +static uint32_t get_wqe_ext_sge_cnt(struct udma_qp *qp) +{ + /* UD QP only has extended sge */ + if (qp->qp_type == QPT_UD) + return qp->sq.max_gs; + + if (qp->sq.max_gs > UDMA_SGE_IN_WQE) + return qp->sq.max_gs - UDMA_SGE_IN_WQE; + + return 0; +} + +static void set_ext_sge_param(struct udma_dev *udma_dev, uint32_t sq_wqe_cnt, + struct udma_qp *qp, struct udma_qp_cap *cap) +{ + uint32_t max_inline_data; + uint32_t total_sge_cnt; + uint32_t ext_sge_cnt; + uint32_t wqe_sge_cnt; + + qp->sge.sge_shift = UDMA_SGE_SHIFT; + + max_inline_data = roundup_pow_of_two(cap->max_inline_data); + ext_sge_cnt = max_inline_data / UDMA_SGE_SIZE; + + /* Select the max data set by the user */ + qp->sq.max_gs = max(ext_sge_cnt, cap->max_send_sge); + + wqe_sge_cnt = get_wqe_ext_sge_cnt(qp); + /* If the number of extended sge is not zero, they MUST use the + * space of UDMA_EP_PAGE_SIZE at least. + */ + if (wqe_sge_cnt) { + total_sge_cnt = roundup_pow_of_two(sq_wqe_cnt * wqe_sge_cnt); + qp->sge.sge_cnt = max(total_sge_cnt, + (uint32_t)UDMA_PAGE_SIZE / UDMA_SGE_SIZE); + } + + /* Ensure that the max_gs size does not exceed */ + qp->sq.max_gs = min(qp->sq.max_gs, udma_dev->caps.max_sq_sg); +} + +static void set_rq_size(struct udma_qp *qp, struct udma_qp_cap *cap) +{ + /* set rq param to 0 */ + qp->rq.wqe_cnt = 0; + qp->rq.max_gs = 1; + cap->max_recv_wr = 0; + cap->max_recv_sge = 0; +} + +static int set_user_sq_size(struct udma_dev *udma_dev, struct udma_qp *qp, + struct udma_qp_cap *cap) +{ + uint32_t cfg_depth; + + if (cap->max_send_wr > udma_dev->caps.max_wqes || + cap->max_send_sge > udma_dev->caps.max_sq_sg) + return -EINVAL; + + qp->sq.wqe_shift = UDMA_SQ_WQE_SHIFT; + cfg_depth = roundup_pow_of_two(cap->max_send_wr); + qp->sq.wqe_cnt = cfg_depth < UDMA_MIN_JFS_DEPTH ? + UDMA_MIN_JFS_DEPTH : cfg_depth; + + set_ext_sge_param(udma_dev, qp->sq.wqe_cnt, qp, cap); + + return 0; +} + +static bool is_rc_jetty(struct udma_qp_attr *qp_attr) +{ + if (qp_attr->is_jetty && qp_attr->jetty && + qp_attr->jetty->tp_mode == UBCORE_TP_RC) + return true; + + return false; +} + +static int set_qp_param(struct udma_dev *udma_dev, struct udma_qp *qp, + struct ubcore_udata *udata, + struct udma_create_tp_ucmd *ucmd) +{ + struct udma_qp_attr *qp_attr = &qp->qp_attr; + struct device *dev = udma_dev->dev; + int ret = 0; + + qp->qp_type = qp_attr->qp_type; + + if (!qp_attr->is_tgt) { + qp->retry_cnt = qp_attr->cap.retry_cnt; + qp->ack_timeout = qp_attr->cap.ack_timeout; + qp->rnr_retry = qp_attr->cap.rnr_retry; + if (qp_attr->is_jetty) + qp->min_rnr_timer = qp_attr->cap.min_rnr_timer; + qp->priority = qp_attr->priority; + } else { + qp->min_rnr_timer = qp_attr->cap.min_rnr_timer; + if (qp_attr->is_jetty) { + qp->retry_cnt = qp_attr->cap.retry_cnt; + qp->ack_timeout = qp_attr->cap.ack_timeout; + qp->rnr_retry = qp_attr->cap.rnr_retry; + qp->priority = qp_attr->priority; + } + } + + if (qp_attr->cap.max_inline_data > udma_dev->caps.max_sq_inline) + qp_attr->cap.max_inline_data = udma_dev->caps.max_sq_inline; + + qp->max_inline_data = qp_attr->cap.max_inline_data; + + set_rq_size(qp, &qp_attr->cap); + + if (udata && udata->uctx != NULL) { + ret = copy_from_user(ucmd, (void *)udata->udrv_data->in_addr, + min(udata->udrv_data->in_len, + (uint32_t)sizeof(struct udma_create_tp_ucmd))); + if (ret) { + dev_err(dev, "failed to copy create tp ucmd\n"); + return ret; + } + + ret = set_user_sq_size(udma_dev, qp, &qp_attr->cap); + if (ret) + dev_err(dev, + "failed to set user SQ size, ret = %d.\n", ret); + } else { + if (is_rc_jetty(qp_attr)) { + ret = set_user_sq_size(udma_dev, qp, &qp_attr->cap); + if (ret) + dev_err(dev, + "failed to set user SQ size for RC Jetty, ret = %d.\n", + ret); + } + } + + return ret; +} + +static uint8_t get_least_load_bankid_for_qp(struct udma_bank *bank) +{ + uint32_t least_load = bank[0].inuse; + uint8_t bankid = 0; + uint32_t bankcnt; + uint8_t i; + + for (i = 1; i < UDMA_QP_BANK_NUM; i++) { + bankcnt = bank[i].inuse; + if (bankcnt < least_load) { + least_load = bankcnt; + bankid = i; + } + } + + return bankid; +} + +static int alloc_qpn_with_bankid(struct udma_bank *bank, uint8_t bankid, + uint64_t *qpn) +{ + int idx; + + idx = ida_alloc_range(&bank->ida, bank->next, bank->max, GFP_KERNEL); + if (idx < 0) { + idx = ida_alloc_range(&bank->ida, bank->min, bank->max, + GFP_KERNEL); + if (idx < 0) + return idx; + } + + bank->next = + ((uint32_t)idx + 1) > bank->max ? bank->min : (uint32_t)idx + 1; + + /* the lower 3 bits is bankid */ + *qpn = (idx << 3) | bankid; + + return 0; +} + +static int alloc_qpn(struct udma_dev *udma_dev, struct udma_qp *qp) +{ + struct udma_qpn_bitmap *qpn_map = qp->qp_attr.qpn_map; + struct device *dev = udma_dev->dev; + uint64_t num = 0; + uint8_t bankid; + int ret; + + if (qpn_map->qpn_shift == 0 || qp->qp_type == QPT_UD) { + qp->qpn = gen_qpn(qpn_map->qpn_prefix, + qpn_map->jid << qpn_map->qpn_shift, 0); + } else { + mutex_lock(&qpn_map->bank_mutex); + bankid = get_least_load_bankid_for_qp(qpn_map->bank); + ret = alloc_qpn_with_bankid(&qpn_map->bank[bankid], bankid, + &num); + if (ret) { + dev_err(dev, "failed to alloc QPN, ret = %d\n", ret); + mutex_unlock(&qpn_map->bank_mutex); + return ret; + } + qpn_map->bank[bankid].inuse++; + mutex_unlock(&qpn_map->bank_mutex); + qp->qpn = gen_qpn(qpn_map->qpn_prefix, + qpn_map->jid << qpn_map->qpn_shift, num); + } + atomic_inc(&qpn_map->ref_num); + + return 0; +} + +static void init_qpn_bitmap(struct udma_qpn_bitmap *qpn_map, uint32_t qpn_shift) +{ + int i; + + qpn_map->qpn_shift = qpn_shift; + mutex_init(&qpn_map->bank_mutex); + /* reserved 0 for UD */ + qpn_map->bank[0].min = 1; + qpn_map->bank[0].inuse = 1; + qpn_map->bank[0].next = qpn_map->bank[0].min; + for (i = 0; i < UDMA_QP_BANK_NUM; i++) { + ida_init(&qpn_map->bank[i].ida); + qpn_map->bank[i].max = (1 << qpn_shift) / UDMA_QP_BANK_NUM - 1; + } +} + +void init_jetty_x_qpn_bitmap(struct udma_dev *dev, + struct udma_qpn_bitmap *qpn_map, + uint32_t jetty_x_shift, + uint32_t prefix, uint32_t jid) +{ +#define QPN_SHIFT_MIN 3 + int qpn_shift; + + qpn_shift = dev->caps.num_qps_shift - jetty_x_shift - + UDMA_JETTY_X_PREFIX_BIT_NUM; + if (qpn_shift <= QPN_SHIFT_MIN) { + qpn_map->qpn_shift = 0; + return; + } + + qpn_map->qpn_prefix = prefix << + (dev->caps.num_qps_shift - + UDMA_JETTY_X_PREFIX_BIT_NUM); + qpn_map->jid = jid; + init_qpn_bitmap(qpn_map, qpn_shift); +} + +void clean_jetty_x_qpn_bitmap(struct udma_qpn_bitmap *qpn_map) +{ + int i; + + if (!qpn_map->qpn_shift) + return; + mutex_lock(&qpn_map->bank_mutex); + for (i = 0; i < UDMA_QP_BANK_NUM; i++) + ida_destroy(&qpn_map->bank[i].ida); + mutex_unlock(&qpn_map->bank_mutex); +} + +static int set_wqe_buf_attr(struct udma_dev *udma_dev, struct udma_qp *qp, + struct udma_buf_attr *buf_attr, bool dca_en) +{ + uint32_t idx = 0; + int buf_size; + + qp->buff_size = 0; + + /* SQ WQE */ + qp->sq.offset = 0; + + buf_size = to_udma_hem_entries_size(qp->sq.wqe_cnt, + qp->sq.wqe_shift); + if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) { + buf_attr->region[idx].size = buf_size; + buf_attr->region[idx].hopnum = udma_dev->caps.wqe_sq_hop_num; + idx++; + qp->buff_size += buf_size; + } + /* extend SGE WQE in SQ */ + qp->sge.offset = qp->buff_size; + + buf_size = to_udma_hem_entries_size(qp->sge.sge_cnt, + qp->sge.sge_shift); + if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) { + buf_attr->region[idx].size = buf_size; + buf_attr->region[idx].hopnum = udma_dev->caps.wqe_sge_hop_num; + idx++; + qp->buff_size += buf_size; + } + + if (qp->buff_size < 1) + return -EINVAL; + + buf_attr->region_count = idx; + + if (dca_en) { + /* When enable DCA, there's no need to alloc buffer now, and + * the page shift should be fixed to 4K. + */ + buf_attr->mtt_only = true; + buf_attr->page_shift = UDMA_HW_PAGE_SHIFT; + } else { + buf_attr->mtt_only = false; + buf_attr->page_shift = UDMA_HW_PAGE_SHIFT + + udma_dev->caps.mtt_buf_pg_sz; + } + + return 0; +} + +static int alloc_wqe_buf(struct udma_dev *dev, struct udma_qp *qp, + struct udma_buf_attr *buf_attr, uint64_t addr, + bool dca_en) +{ + int ret; + + if (dca_en) { + /* DCA must be enabled after the buffer attr is configured. */ + udma_enable_dca(dev, qp); + qp->en_flags |= UDMA_QP_CAP_DYNAMIC_CTX_ATTACH; + } else if ((PAGE_SIZE <= UDMA_DWQE_SIZE) && + (dev->caps.flags & UDMA_CAP_FLAG_DIRECT_WQE) && + (qp->qpn < UDMA_DWQE_MMAP_QP_NUM)) { + qp->en_flags |= UDMA_QP_CAP_DIRECT_WQE; + } + + ret = udma_mtr_create(dev, &qp->mtr, buf_attr, + PAGE_SHIFT + dev->caps.mtt_ba_pg_sz, addr, true); + if (ret) { + dev_err(dev->dev, "failed to create WQE mtr, ret = %d.\n", ret); + if (dca_en) + udma_disable_dca(dev, qp); + } + + return ret; +} + +static bool check_dca_is_enable(struct udma_dev *udma_dev, struct udma_qp *qp, + uint64_t buf_addr) +{ + if (qp->qp_type != QPT_RC || + !(udma_dev->caps.flags & UDMA_CAP_FLAG_DCA_MODE)) + return false; + + /* If the user QP's buffer addr is 0, the DCA mode should be enabled */ + return !buf_addr; +} + +static int alloc_qp_wqe(struct udma_dev *udma_dev, struct udma_qp *qp, + uint64_t buf_addr) +{ + struct device *dev = udma_dev->dev; + struct udma_buf_attr buf_attr = {}; + bool dca_en; + int ret; + + dca_en = check_dca_is_enable(udma_dev, qp, buf_addr); + ret = set_wqe_buf_attr(udma_dev, qp, &buf_attr, dca_en); + if (ret) { + dev_err(dev, "failed to set WQE attr, ret = %d.\n", ret); + return ret; + } + + ret = alloc_wqe_buf(udma_dev, qp, &buf_attr, buf_addr, dca_en); + if (ret) { + dev_err(dev, "failed to alloc WQE buf, ret = %d.\n", ret); + return ret; + } + + return 0; +} + +static int alloc_user_qp_db(struct udma_dev *udma_dev, struct udma_qp *qp, + struct udma_create_tp_ucmd *ucmd) +{ + int ret; + + if (!ucmd->sdb_addr) + return 0; + + ret = udma_db_map_user(udma_dev, ucmd->sdb_addr, &qp->sdb); + if (ret) { + dev_err(udma_dev->dev, + "failed to map user sdb_addr, ret = %d.\n", ret); + return ret; + } + + qp->en_flags |= UDMA_QP_CAP_SQ_RECORD_DB; + + return 0; +} + +static int alloc_qp_db(struct udma_dev *udma_dev, struct udma_qp *qp, + struct ubcore_udata *udata, + struct udma_create_tp_ucmd *ucmd) +{ + int ret; + + if (udma_dev->caps.flags & UDMA_CAP_FLAG_SDI_MODE) + qp->en_flags |= UDMA_QP_CAP_OWNER_DB; + + if (udata) { + ret = alloc_user_qp_db(udma_dev, qp, ucmd); + if (ret) + return ret; + } + + return 0; +} + +static int alloc_qpc(struct udma_dev *udma_dev, struct udma_qp *qp) +{ + struct udma_qp_table *qp_table = &udma_dev->qp_table; + struct device *dev = udma_dev->dev; + int ret; + + /* Alloc memory for QPC */ + ret = udma_table_get(udma_dev, &qp_table->qp_table, qp->qpn); + if (ret) { + dev_err(dev, "Failed to get QPC table\n"); + goto err_out; + } + + /* Alloc memory for IRRL */ + ret = udma_table_get(udma_dev, &qp_table->irrl_table, qp->qpn); + if (ret) { + dev_err(dev, "Failed to get IRRL table\n"); + goto err_put_qp; + } + + if (udma_dev->caps.trrl_entry_sz) { + /* Alloc memory for TRRL */ + ret = udma_table_get(udma_dev, &qp_table->trrl_table, + qp->qpn); + if (ret) { + dev_err(dev, "Failed to get TRRL table\n"); + goto err_put_irrl; + } + } + + if (udma_dev->caps.flags & UDMA_CAP_FLAG_QP_FLOW_CTRL) { + /* Alloc memory for SCC CTX */ + ret = udma_table_get(udma_dev, &qp_table->sccc_table, + qp->qpn); + if (ret) { + dev_err(dev, "Failed to get SCC CTX table\n"); + goto err_put_trrl; + } + } + + if (udma_dev->caps.reorder_cq_buffer_en) { + ret = udma_alloc_reorder_cq_buf(udma_dev, &qp->qp_attr); + if (ret) + dev_warn(udma_dev->dev, + "failed to alloc reorder cq buffer.\n"); + } + + return 0; + +err_put_trrl: + if (udma_dev->caps.trrl_entry_sz) + udma_table_put(udma_dev, &qp_table->trrl_table, qp->qpn); +err_put_irrl: + udma_table_put(udma_dev, &qp_table->irrl_table, qp->qpn); +err_put_qp: + udma_table_put(udma_dev, &qp_table->qp_table, qp->qpn); +err_out: + return ret; +} + +static void udma_lock_cqs(struct udma_jfc *send_jfc, struct udma_jfc *recv_jfc) + __acquires(&send_jfc->lock) + __acquires(&recv_jfc->lock) +{ + if (unlikely(send_jfc == NULL && recv_jfc == NULL)) { + __acquire(&send_jfc->lock); + __acquire(&recv_jfc->lock); + } else if (unlikely(send_jfc != NULL && recv_jfc == NULL)) { + spin_lock_irq(&send_jfc->lock); + __acquire(&recv_jfc->lock); + } else if (unlikely(send_jfc == NULL && recv_jfc != NULL)) { + spin_lock_irq(&recv_jfc->lock); + __acquire(&send_jfc->lock); + } else if (send_jfc == recv_jfc) { + spin_lock_irq(&send_jfc->lock); + __acquire(&recv_jfc->lock); + } else if (send_jfc->cqn < recv_jfc->cqn) { + spin_lock_irq(&send_jfc->lock); + spin_lock_nested(&recv_jfc->lock, SINGLE_DEPTH_NESTING); + } else { + spin_lock_irq(&recv_jfc->lock); + spin_lock_nested(&send_jfc->lock, SINGLE_DEPTH_NESTING); + } +} + +static void udma_unlock_cqs(struct udma_jfc *send_jfc, + struct udma_jfc *recv_jfc) + __releases(&send_jfc->lock) + __releases(&recv_jfc->lock) +{ + if (unlikely(send_jfc == NULL && recv_jfc == NULL)) { + __release(&recv_jfc->lock); + __release(&send_jfc->lock); + } else if (unlikely(send_jfc != NULL && recv_jfc == NULL)) { + __release(&recv_jfc->lock); + spin_unlock(&send_jfc->lock); + } else if (unlikely(send_jfc == NULL && recv_jfc != NULL)) { + __release(&send_jfc->lock); + spin_unlock(&recv_jfc->lock); + } else if (send_jfc == recv_jfc) { + __release(&recv_jfc->lock); + spin_unlock_irq(&send_jfc->lock); + } else if (send_jfc->cqn < recv_jfc->cqn) { + spin_unlock(&recv_jfc->lock); + spin_unlock_irq(&send_jfc->lock); + } else { + spin_unlock(&send_jfc->lock); + spin_unlock_irq(&recv_jfc->lock); + } +} + +void copy_send_jfc(struct udma_qp *from_qp, struct udma_qp *to_qp) +{ + to_qp->qp_attr.send_jfc = from_qp->qp_attr.send_jfc; + udma_lock_cqs(to_qp->qp_attr.send_jfc, NULL); + list_add_tail(&to_qp->sq_node, &to_qp->qp_attr.send_jfc->sq_list); + udma_unlock_cqs(to_qp->qp_attr.send_jfc, NULL); +} + +static void add_qp_to_list(struct udma_dev *udma_dev, struct udma_qp *qp, + struct udma_jfc *send_jfc, struct udma_jfc *recv_jfc) +{ + unsigned long flags; + + spin_lock_irqsave(&udma_dev->qp_list_lock, flags); + udma_lock_cqs(send_jfc, recv_jfc); + + list_add_tail(&qp->node, &udma_dev->qp_list); + + if (send_jfc) + list_add_tail(&qp->sq_node, &send_jfc->sq_list); + if (recv_jfc) + list_add_tail(&qp->rq_node, &recv_jfc->rq_list); + + udma_unlock_cqs(send_jfc, recv_jfc); + spin_unlock_irqrestore(&udma_dev->qp_list_lock, flags); +} + +static int udma_qp_store(struct udma_dev *udma_dev, + struct udma_qp *qp) +{ + struct udma_qp_attr *qp_attr = &qp->qp_attr; + struct xarray *xa = &udma_dev->qp_table.xa; + int ret; + + ret = xa_err(xa_store_irq(xa, qp->qpn, qp, GFP_KERNEL)); + if (ret) + dev_err(udma_dev->dev, "Failed to xa store for QPC\n"); + else + /* add QP to device's QP list for softwc */ + add_qp_to_list(udma_dev, qp, qp_attr->send_jfc, + qp_attr->recv_jfc); + + return ret; +} + +static void udma_qp_remove(struct udma_dev *udma_dev, struct udma_qp *qp) +{ + struct udma_qp_attr *qp_attr = &qp->qp_attr; + struct xarray *xa = &udma_dev->qp_table.xa; + struct udma_jfc *send_jfc; + struct udma_jfc *recv_jfc; + unsigned long flags; + + send_jfc = qp_attr->send_jfc; + recv_jfc = qp_attr->recv_jfc; + + xa_lock_irqsave(xa, flags); + __xa_erase(xa, qp->qpn); + xa_unlock_irqrestore(xa, flags); + + spin_lock_irqsave(&udma_dev->qp_list_lock, flags); + udma_lock_cqs(send_jfc, recv_jfc); + + list_del(&qp->node); + + if (send_jfc) + list_del(&qp->sq_node); + if (recv_jfc) + list_del(&qp->rq_node); + + udma_unlock_cqs(send_jfc, recv_jfc); + spin_unlock_irqrestore(&udma_dev->qp_list_lock, flags); +} + +static void free_qpc(struct udma_dev *udma_dev, struct udma_qp *qp) +{ + struct udma_qp_table *qp_table = &udma_dev->qp_table; + + if (udma_dev->caps.reorder_cq_buffer_en) + udma_free_reorder_cq_buf(udma_dev, &qp->qp_attr); + + if (udma_dev->caps.flags & UDMA_CAP_FLAG_QP_FLOW_CTRL) { + udma_table_put(udma_dev, &qp_table->sccc_table, qp->qpn); + + if (qp->dip_idx != qp->qpn && qp->dip_idx >= 0) + udma_table_put(udma_dev, &qp_table->sccc_table, + qp->dip_idx); + } + + if (udma_dev->caps.trrl_entry_sz) + udma_table_put(udma_dev, &qp_table->trrl_table, qp->qpn); + + udma_table_put(udma_dev, &qp_table->irrl_table, qp->qpn); +} + +static void free_qp_db(struct udma_dev *udma_dev, struct udma_qp *qp) +{ + if ((is_rc_jetty(&qp->qp_attr) && + !(qp->en_flags & UDMA_QP_CAP_DYNAMIC_CTX_ATTACH)) || + qp->no_free_wqe_buf) + return; + + if (qp->en_flags & UDMA_QP_CAP_SQ_RECORD_DB) + udma_db_unmap_user(udma_dev, &qp->sdb); +} + +static void free_wqe_buf(struct udma_dev *dev, struct udma_qp *qp) +{ + if ((is_rc_jetty(&qp->qp_attr) && + !(qp->en_flags & UDMA_QP_CAP_DYNAMIC_CTX_ATTACH)) || + qp->no_free_wqe_buf) + return; + + udma_mtr_destroy(dev, &qp->mtr); + if (qp->en_flags & UDMA_QP_CAP_DYNAMIC_CTX_ATTACH) + udma_disable_dca(dev, qp); +} + +static void free_qp_wqe(struct udma_dev *udma_dev, struct udma_qp *qp) +{ + free_wqe_buf(udma_dev, qp); +} + +static inline uint8_t get_qp_bankid(uint64_t qpn) +{ + /* The lower 3 bits of QPN are used to hash to different banks */ + return (uint8_t)(qpn & QP_BANKID_MASK); +} + +static void free_qpn(struct udma_qp *qp) +{ + struct udma_qpn_bitmap *qpn_map = qp->qp_attr.qpn_map; + uint8_t bankid; + + if (qpn_map->qpn_shift == 0 || qp->qp_type == QPT_UD) + return; + + bankid = get_qp_bankid(qp->qpn); + + mutex_lock(&qpn_map->bank_mutex); + if (!ida_is_empty(&qpn_map->bank[bankid].ida)) { + ida_free(&qpn_map->bank[bankid].ida, + (qp->qpn & GENMASK(qpn_map->qpn_shift - 1, 0)) >> + QP_BANKID_SHIFT); + } + qpn_map->bank[bankid].inuse--; + mutex_unlock(&qpn_map->bank_mutex); +} + +bool udma_qp_need_alloc_sq(struct udma_qp_attr *qp_attr) +{ + if (qp_attr->is_jetty) { + /* create qp for UM jetty */ + if (qp_attr->jetty->tp_mode == UBCORE_TP_UM) + return true; + + if (qp_attr->jetty->tp_mode == UBCORE_TP_RC) + return true; + + return !qp_attr->is_tgt; + } + + /* create qp for jfs for send */ + if (qp_attr->jfs) + return true; + + /* create qp for jfr for recv */ + return false; +} + +static uint32_t udma_get_jetty_qpn(struct udma_qp *qp) +{ + struct udma_tp *tp = container_of(qp, struct udma_tp, qp); + struct udma_jetty *jetty = qp->qp_attr.jetty; + uint32_t qpn = qp->qpn; + struct udma_tp *pre_tp; + uint32_t hash; + + if (jetty->tp_mode == UBCORE_TP_RC && jetty->rc_node.tp != NULL) + qpn = jetty->rc_node.tpn; + + if (jetty->tp_mode == UBCORE_TP_RM) { + hash = udma_get_jetty_hash(&tp->tjetty_id); + pre_tp = + (struct udma_tp *)xa_load(&jetty->srm_node_table, hash); + if (pre_tp) + qpn = pre_tp->qp.qpn; + } + + return qpn; +} + +int udma_create_qp_common(struct udma_dev *udma_dev, struct udma_qp *qp, + struct ubcore_udata *udata) +{ + struct udma_ucontext *uctx = to_udma_ucontext(udata->uctx); + struct udma_qp_attr *qp_attr = &qp->qp_attr; + struct udma_qp_context ctx[2] = {0}; + struct device *dev = udma_dev->dev; + struct udma_create_tp_ucmd ucmd; + struct udma_create_tp_resp resp; + bool udma_alloc_sq_flag = false; + int ret; + + qp->state = QPS_RESET; + qp->dip_idx = UDMA_SCC_DIP_INVALID_IDX; + qp->dca_ctx = udata->uctx ? &uctx->dca_ctx : qp->dca_ctx; + + ret = set_qp_param(udma_dev, qp, udata, &ucmd); + if (ret) { + dev_err(dev, "failed to set QP param, ret = %d.\n", ret); + return ret; + } + + ret = alloc_qpn(udma_dev, qp); + if (ret) { + dev_err(dev, "failed to alloc QPN, ret = %d.\n", ret); + goto err_qpn; + } + + if (!qp->qpn) { + ret = -EINVAL; + goto err_qpn; + } + + if (udma_qp_need_alloc_sq(qp_attr)) { + udma_alloc_sq_flag = is_rc_jetty(qp_attr) && (ucmd.buf_addr || qp_attr->is_tgt); + if (udma_alloc_sq_flag) { + qp->mtr = qp_attr->jetty->rc_node.mtr; + qp->sdb = qp_attr->jetty->rc_node.sdb; + qp->en_flags |= UDMA_QP_CAP_SQ_RECORD_DB; + } else { + ret = alloc_qp_wqe(udma_dev, qp, ucmd.buf_addr); + if (ret) { + dev_err(dev, + "failed to alloc QP buffer, ret = %d.\n", + ret); + goto err_buf; + } + + ret = alloc_qp_db(udma_dev, qp, udata, &ucmd); + if (ret) { + dev_err(dev, + "failed to alloc QP doorbell, ret = %d.\n", + ret); + goto err_db; + } + } + } + + ret = alloc_qpc(udma_dev, qp); + if (ret) { + dev_err(dev, "failed to alloc QP context, ret = %d.\n", + ret); + goto err_qpc; + } + + ret = udma_qp_store(udma_dev, qp); + if (ret) { + dev_err(dev, "failed to store QP, ret = %d.\n", ret); + goto err_store; + } + + if (udata && udata->uctx) { + resp.cap_flags = qp->en_flags; + resp.qpn = qp->qpn; + resp.priority = qp->priority; + if (qp_attr->is_jetty && qp_attr->jetty) + resp.qpn = udma_get_jetty_qpn(qp); + + resp.path_mtu = udma_dev->caps.max_mtu; + resp.um_srcport.um_spray_en = um_spray_en; + resp.um_srcport.um_data_udp_start = (uint16_t)um_data_udp_start; + resp.um_srcport.um_udp_range = (uint8_t)um_udp_range + + UDP_RANGE_BASE; + ret = copy_to_user((void *)udata->udrv_data->out_addr, &resp, + min(udata->udrv_data->out_len, + (uint32_t)sizeof(resp))); + if (ret) { + dev_err(dev, "copy qp resp failed!\n"); + goto err_copy; + } + } + + refcount_set(&qp->refcount, 1); + init_completion(&qp->free); + + ret = udma_pass_qpc_to_hw(udma_dev, ctx, ctx+1, qp); + if (ret) { + dev_err(dev, "failed to pass QPC to HW, ret = %d.\n", ret); + goto err_copy; + } + + return 0; + +err_copy: + udma_qp_remove(udma_dev, qp); +err_store: + free_qpc(udma_dev, qp); +err_qpc: + if (udma_qp_need_alloc_sq(&qp->qp_attr)) + free_qp_db(udma_dev, qp); +err_db: + if (udma_qp_need_alloc_sq(&qp->qp_attr)) + free_qp_wqe(udma_dev, qp); +err_buf: + free_qpn(qp); +err_qpn: + return ret; +} + +void udma_destroy_qp_common(struct udma_dev *udma_dev, struct udma_qp *qp) +{ + udma_qp_remove(udma_dev, qp); + + if (refcount_dec_and_test(&qp->refcount)) + complete(&qp->free); + wait_for_completion(&qp->free); + + free_qpc(udma_dev, qp); + if (udma_qp_need_alloc_sq(&qp->qp_attr) || qp->force_free_wqe_buf) { + free_qp_db(udma_dev, qp); + free_qp_wqe(udma_dev, qp); + } + free_qpn(qp); +} + +int udma_init_qp_table(struct udma_dev *dev) +{ + struct udma_qp_table *qp_table = &dev->qp_table; + uint32_t reserved_from_bot; + uint32_t i; + + qp_table->idx_table.spare_idx = kcalloc(dev->caps.num_qps, + sizeof(uint32_t), GFP_KERNEL); + if (!qp_table->idx_table.spare_idx) + return -ENOMEM; + + mutex_init(&qp_table->bank_mutex); + xa_init(&qp_table->xa); + + reserved_from_bot = dev->caps.reserved_qps; + + for (i = 0; i < reserved_from_bot; i++) { + dev->qp_table.bank[get_qp_bankid(i)].inuse++; + dev->qp_table.bank[get_qp_bankid(i)].min++; + } + + for (i = 0; i < UDMA_QP_BANK_NUM; i++) { + ida_init(&dev->qp_table.bank[i].ida); + dev->qp_table.bank[i].max = dev->caps.num_qps / + UDMA_QP_BANK_NUM - 1; + dev->qp_table.bank[i].next = dev->qp_table.bank[i].min; + } + + return 0; +} + +void udma_cleanup_qp_table(struct udma_dev *dev) +{ + int i; + + for (i = 0; i < UDMA_QP_BANK_NUM; i++) + ida_destroy(&dev->qp_table.bank[i].ida); + kfree(dev->qp_table.idx_table.spare_idx); +} + +int udma_flush_cqe(struct udma_dev *udma_dev, struct udma_qp *udma_qp, + uint32_t sq_pi) +{ + struct udma_qp_context *qp_context; + struct udma_cmd_mailbox *mailbox; + struct udma_qp_context *qpc_mask; + struct udma_cmq_desc desc; + struct udma_mbox *mb; + int ret; + + udma_qp->state = QPS_ERR; + + mailbox = udma_alloc_cmd_mailbox(udma_dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + qp_context = (struct udma_qp_context *)mailbox->buf; + qpc_mask = (struct udma_qp_context *)mailbox->buf + 1; + memset(qpc_mask, 0xff, sizeof(struct udma_qp_context)); + + udma_reg_write(qp_context, QPC_QP_ST, udma_qp->state); + udma_reg_clear(qpc_mask, QPC_QP_ST); + + udma_reg_write(qp_context, QPC_SQ_PRODUCER_IDX, sq_pi); + udma_reg_clear(qpc_mask, QPC_SQ_PRODUCER_IDX); + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mbox_desc_init(mb, mailbox->dma, 0, udma_qp->qpn, UDMA_CMD_MODIFY_QPC); + + ret = udma_cmd_mbox(udma_dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); + if (ret) + dev_err(udma_dev->dev, "flush cqe qp(0x%llx) cmd error(%d).\n", + udma_qp->qpn, ret); + + udma_free_cmd_mailbox(udma_dev, mailbox); + + return ret; +} + +void udma_qp_event(struct udma_dev *udma_dev, uint32_t qpn, int event_type) +{ + struct device *dev = udma_dev->dev; + struct udma_qp *qp; + + xa_lock(&udma_dev->qp_table.xa); + qp = (struct udma_qp *)xa_load(&udma_dev->qp_table.xa, qpn); + if (qp) + refcount_inc(&qp->refcount); + xa_unlock(&udma_dev->qp_table.xa); + + if (!qp) { + dev_warn(dev, "Async event for bogus QP 0x%08x\n", qpn); + return; + } + + if (event_type == UDMA_EVENT_TYPE_JFR_LAST_WQE_REACH || + event_type == UDMA_EVENT_TYPE_WQ_CATAS_ERROR || + event_type == UDMA_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR || + event_type == UDMA_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR) { + qp->state = QPS_ERR; + + if (qp->sdb.virt_addr) + qp->sq.head = *(int *)(qp->sdb.virt_addr); + + udma_flush_cqe(udma_dev, qp, qp->sq.head); + } + + if (qp->event) + qp->event(qp, (enum udma_event)event_type); + + if (refcount_dec_and_test(&qp->refcount)) + complete(&qp->free); +} + +module_param(um_spray_en, bool, 0644); +MODULE_PARM_DESC(um_spray_en, + "Set whether to enable the multipath function for UM Jetty/Jfs, default: 0(0:off, 1:on)"); + +module_param(um_data_udp_start, ushort, 0644); +MODULE_PARM_DESC(um_data_udp_start, + "Set the Initial source port number for UM Jetty/Jfs, valid when um_spray_en is set 1"); + +module_param(um_udp_range, ushort, 0644); +MODULE_PARM_DESC(um_udp_range, + "Set the variable bits of source port number for UM Jetty/Jfs, valid when um_spray_en is set 1, range:0-8, default: 0. 0 ~ (7 + um_udp_range) bits of source port are variable"); diff --git a/drivers/ub/hw/hns3/hns3_udma_qp.h b/drivers/ub/hw/hns3/hns3_udma_qp.h new file mode 100644 index 0000000000000000000000000000000000000000..24a1d8a2e50af44bea5e8d6ad1fc7fa82fe3ae33 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_qp.h @@ -0,0 +1,358 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_QP_H +#define _UDMA_QP_H + +#include "hns3_udma_device.h" + +#define UDMA_GID_SIZE 16 +#define SGEN_INI_VALUE 3 +#define QP_TIMEOUT_MAX 31 +#define WQE_SGE_BA_OFFSET 3 +#define H_ADDR_OFFSET 32 +#define MAX_LP_MSG_LEN 16384 +#define QPC_DMAC_H_IDX 4 +#define UDP_RANGE_BASE 8 +#define UDMA_SQ_WQE_SHIFT 6 +#define RETRY_MSG_PSN_H_OFFSET 16 + +struct udma_qp_context_ex { + uint32_t data[64]; +}; + +struct udma_qp_context { + uint32_t qpc_context1; + uint32_t wqe_sge_ba; + uint32_t qpc_context2[5]; + uint8_t dgid[UDMA_GID_SIZE]; + uint32_t dmac; + uint32_t qpc_context3[3]; + uint32_t qkey_xrcd; + uint32_t qpc_context4[11]; + uint32_t rq_rnr_timer; + uint32_t qpc_context5[36]; + struct udma_qp_context_ex ext; +}; + +#define QPC_FIELD_LOC(h, l) ((uint64_t)(h) << 32 | (l)) + +#define QPC_TST QPC_FIELD_LOC(2, 0) +#define QPC_SGE_SHIFT QPC_FIELD_LOC(7, 3) +#define QPC_WQE_SGE_BA_L QPC_FIELD_LOC(63, 32) +#define QPC_WQE_SGE_BA_H QPC_FIELD_LOC(92, 64) +#define QPC_SQ_HOP_NUM QPC_FIELD_LOC(94, 93) +#define QPC_WQE_SGE_BA_PG_SZ QPC_FIELD_LOC(99, 96) +#define QPC_WQE_SGE_BUF_PG_SZ QPC_FIELD_LOC(103, 100) +#define QPC_SGE_HOP_NUM QPC_FIELD_LOC(131, 130) +#define QPC_RQWS QPC_FIELD_LOC(135, 132) +#define QPC_SQ_SHIFT QPC_FIELD_LOC(139, 136) +#define QPC_GMV_IDX QPC_FIELD_LOC(159, 144) +#define QPC_HOPLIMIT QPC_FIELD_LOC(167, 160) +#define QPC_VLAN_ID QPC_FIELD_LOC(187, 176) +#define QPC_MTU QPC_FIELD_LOC(191, 188) +#define QPC_FL QPC_FIELD_LOC(211, 192) +#define QPC_SL QPC_FIELD_LOC(215, 212) +#define QPC_AT QPC_FIELD_LOC(223, 219) +#define QPC_DMAC_L QPC_FIELD_LOC(383, 352) +#define QPC_DMAC_H QPC_FIELD_LOC(399, 384) +#define QPC_DQPN QPC_FIELD_LOC(439, 416) +#define QPC_LP_PKTN_INI QPC_FIELD_LOC(447, 444) +#define QPC_CONGEST_ALGO_TMPL_ID QPC_FIELD_LOC(455, 448) +#define QPC_QP_ST QPC_FIELD_LOC(479, 477) +#define QPC_RQ_RECORD_EN QPC_FIELD_LOC(512, 512) +#define QPC_RQ_DB_RECORD_ADDR_L QPC_FIELD_LOC(543, 513) +#define QPC_RQ_DB_RECORD_ADDR_H QPC_FIELD_LOC(575, 544) +#define QPC_SRQN QPC_FIELD_LOC(599, 576) +#define QPC_SRQ_EN QPC_FIELD_LOC(600, 600) +#define QPC_RRE QPC_FIELD_LOC(601, 601) +#define QPC_RWE QPC_FIELD_LOC(602, 602) +#define QPC_RX_CQN QPC_FIELD_LOC(631, 608) +#define QPC_XRC_QP_TYPE QPC_FIELD_LOC(632, 632) +#define QPC_CQEIE QPC_FIELD_LOC(633, 633) +#define QPC_MIN_RNR_TIME QPC_FIELD_LOC(639, 635) +#define QPC_FLUSH_EN QPC_FIELD_LOC(821, 821) +#define QPC_AW_EN QPC_FIELD_LOC(822, 822) +#define QPC_WN_EN QPC_FIELD_LOC(823, 823) +#define QPC_INV_CREDIT QPC_FIELD_LOC(832, 832) +#define QPC_RX_REQ_EPSN QPC_FIELD_LOC(863, 840) +#define QPC_RR_MAX QPC_FIELD_LOC(1102, 1100) +#define QPC_RAQ_PSN QPC_FIELD_LOC(1207, 1184) +#define QPC_SQ_PRODUCER_IDX QPC_FIELD_LOC(1263, 1248) +#define QPC_SQ_CONSUMER_IDX QPC_FIELD_LOC(1279, 1264) +#define QPC_SQ_CUR_BLK_ADDR_L QPC_FIELD_LOC(1311, 1280) +#define QPC_SQ_CUR_BLK_ADDR_H QPC_FIELD_LOC(1331, 1312) +#define QPC_LP_SGEN_INI QPC_FIELD_LOC(1335, 1334) +#define QPC_ACK_REQ_FREQ QPC_FIELD_LOC(1349, 1344) +#define QPC_SQ_CUR_PSN QPC_FIELD_LOC(1375, 1352) +#define QPC_SQ_CUR_SGE_BLK_ADDR_L QPC_FIELD_LOC(1439, 1408) +#define QPC_SQ_CUR_SGE_BLK_ADDR_H QPC_FIELD_LOC(1459, 1440) +#define QPC_OWNER_MODE QPC_FIELD_LOC(1536, 1536) +#define QPC_DCA_MODE QPC_FIELD_LOC(1542, 1542) +#define QPC_SQ_MAX_PSN QPC_FIELD_LOC(1567, 1544) +#define QPC_RMT_E2E QPC_FIELD_LOC(1660, 1660) +#define QPC_RETRY_NUM_INIT QPC_FIELD_LOC(1690, 1688) +#define QPC_RETRY_CNT QPC_FIELD_LOC(1695, 1693) +#define QPC_RETRY_MSG_MSN QPC_FIELD_LOC(1743, 1728) +#define QPC_RETRY_MSG_PSN_L QPC_FIELD_LOC(1759, 1744) +#define QPC_RETRY_MSG_PSN_H QPC_FIELD_LOC(1767, 1760) +#define QPC_RETRY_MSG_FPKT_PSN QPC_FIELD_LOC(1791, 1768) +#define QPC_RX_SQ_CUR_BLK_ADDR_L QPC_FIELD_LOC(1823, 1792) +#define QPC_RX_SQ_CUR_BLK_ADDR_H QPC_FIELD_LOC(1843, 1824) +#define QPC_RX_ACK_EPSN QPC_FIELD_LOC(1943, 1920) +#define QPC_RNR_NUM_INIT QPC_FIELD_LOC(1946, 1944) +#define QPC_RNR_CNT QPC_FIELD_LOC(1949, 1947) +#define QPC_TX_CQN QPC_FIELD_LOC(2007, 1984) +#define QPC_SIG_TYPE QPC_FIELD_LOC(2008, 2008) + +#define QPCEX_FIELD_LOC(h, l) ((uint64_t)(h) << 32 | (l)) + +#define QPCEX_ON_FLIGHT_SIZE_H_SHIFT 3 +#define QPCEX_REORDER_CQ_ADDR_SHIFT 12 +#define QPCEX_DATA_UDP_SRCPORT_H_SHIFT 11 +#define QPCEX_RTT_INIT 100 +#define QPCEX_P_TYPE_UDMA 0x1 +#define MAX_SERVICE_LEVEL 0x7 + +#define QPCEX_CONGEST_ALG_SEL QPCEX_FIELD_LOC(0, 0) +#define QPCEX_CONGEST_ALG_SUB_SEL QPCEX_FIELD_LOC(1, 1) +#define QPCEX_DIP_CTX_IDX_VLD QPCEX_FIELD_LOC(2, 2) +#define QPCEX_DIP_CTX_IDX QPCEX_FIELD_LOC(22, 3) +#define QPCEX_SQ_RQ_NOT_FORBID_EN QPCEX_FIELD_LOC(23, 23) +#define QPCEX_RTT QPCEX_FIELD_LOC(81, 66) +#define QPCEX_AR_EN QPCEX_FIELD_LOC(112, 112) +#define QPCEX_UDP_SRCPORT_RANGE QPCEX_FIELD_LOC(116, 113) +#define QPCEX_DATA_UDP_SRCPORT_L QPCEX_FIELD_LOC(127, 117) +#define QPCEX_DATA_UDP_SRCPORT_H QPCEX_FIELD_LOC(132, 128) +#define QPCEX_ACK_UDP_SRCPORT QPCEX_FIELD_LOC(148, 133) +#define QPCEX_REORDER_CAP QPCEX_FIELD_LOC(155, 153) +#define QPCEX_ON_FLIGHT_SIZE_L QPCEX_FIELD_LOC(159, 157) +#define QPCEX_ON_FLIGHT_SIZE_H QPCEX_FIELD_LOC(172, 160) +#define QPCEX_OOR_EN QPCEX_FIELD_LOC(266, 266) +#define QPCEX_P_TYPE QPCEX_FIELD_LOC(268, 268) +#define QPCEX_DYN_AT QPCEX_FIELD_LOC(272, 269) +#define QPCEX_DEID_H QPCEX_FIELD_LOC(319, 288) +#define QPCEX_REORDER_CQ_EN QPCEX_FIELD_LOC(427, 427) +#define QPCEX_REORDER_CQ_ADDR_L QPCEX_FIELD_LOC(447, 428) +#define QPCEX_REORDER_CQ_ADDR_H QPCEX_FIELD_LOC(479, 448) +#define QPCEX_REORDER_CQ_SHIFT QPCEX_FIELD_LOC(483, 480) +#define UDP_SRCPORT_RANGE_BASE 7 +#define UDP_SRCPORT_RANGE_SIZE_MASK 0xF + +struct udma_modify_tp_attr { + uint8_t dmac[UBCORE_MAC_BYTES]; + uint32_t dest_qp_num; + uint8_t retry_cnt; + uint8_t rnr_retry; + uint8_t priority; + uint8_t ack_timeout; + uint32_t sq_psn; + uint32_t rq_psn; + uint8_t max_dest_rd_atomic; + uint8_t max_rd_atomic; + int qp_access_flags; + uint8_t min_rnr_timer; + uint32_t qkey; + enum ubcore_mtu path_mtu; + uint8_t hop_limit; + uint8_t dgid[UDMA_GID_SIZE]; + uint8_t dipv4[4]; + uint8_t sgid_index; + uint16_t data_udp_start; + uint16_t ack_udp_start; + uint16_t udp_range; + uint32_t ar_en; + enum udma_cong_type cong_alg; +}; + +struct udma_qp_cap { + uint32_t max_send_wr; + uint32_t max_recv_wr; + uint32_t max_send_sge; + uint32_t max_recv_sge; + uint32_t max_inline_data; + uint8_t retry_cnt; + uint8_t rnr_retry; + uint8_t min_rnr_timer; + uint8_t ack_timeout; +}; + +struct udma_qpn_bitmap { + uint32_t qpn_prefix; + uint32_t jid; + uint32_t qpn_shift; + struct udma_bank bank[UDMA_QP_BANK_NUM]; + struct mutex bank_mutex; + atomic_t ref_num; +}; + +struct udma_qp_attr { + bool is_jetty; + bool is_tgt; + struct ubcore_ucontext *uctx; + struct udma_jfc *send_jfc; + struct udma_jfc *recv_jfc; + struct udma_jfs *jfs; + struct udma_jfr *jfr; + struct udma_jetty *jetty; + struct udma_qp_cap cap; + enum udma_qp_type qp_type; + uint32_t pdn; + struct udma_qpn_bitmap *qpn_map; + void *reorder_cq_page; + int reorder_cq_size; + dma_addr_t reorder_cq_addr; + union ubcore_eid remote_eid; + union ubcore_eid local_eid; + int tgt_id; + uint8_t priority; +}; + +struct udma_wq { + uint32_t wqe_cnt; /* WQE num */ + uint32_t max_gs; + int offset; + int wqe_offset; + int wqe_shift; /* WQE size */ + uint32_t head; +}; + +struct udma_qp_sge { + uint32_t sge_cnt; /* SGE num */ + int offset; + int sge_shift; /* SGE size */ + int wqe_offset; +}; + +struct udma_dca_cfg { + spinlock_t lock; + uint32_t attach_count; + uint32_t buf_id; + uint32_t dcan; + void **buf_list; + uint32_t npages; + uint32_t sq_idx; + bool aging_enable; + struct list_head aging_node; +}; + +struct udma_qp { + struct udma_dev *udma_device; + enum udma_qp_type qp_type; + struct udma_qp_attr qp_attr; + struct udma_wq sq; + struct udma_wq rq; + struct udma_db sdb; + struct udma_jfc *send_jfc; + struct udma_jfc *recv_jfc; + uint64_t en_flags; + struct udma_mtr mtr; + struct udma_dca_cfg dca_cfg; + struct udma_dca_ctx *dca_ctx; + uint32_t buff_size; + enum udma_qp_state state; + void (*event)(struct udma_qp *qp, + enum udma_event event_type); + uint64_t qpn; + + refcount_t refcount; + struct completion free; + struct udma_qp_sge sge; + enum udma_mtu path_mtu; + enum ubcore_mtu ubcore_path_mtu; + uint32_t max_inline_data; + uint8_t sl; + struct list_head node; /* all qps are on a list */ + struct list_head rq_node; /* all recv qps are on a list */ + struct list_head sq_node; /* all send qps are on a list */ + uint8_t retry_cnt; + uint8_t rnr_retry; + uint8_t ack_timeout; + uint8_t min_rnr_timer; + uint8_t priority; + bool no_free_wqe_buf; + bool force_free_wqe_buf; + int64_t dip_idx; + struct udma_modify_tp_attr *m_attr; +}; + +struct udma_congestion_algorithm { + uint8_t congest_type; + uint8_t alg_sel; + uint8_t alg_sub_sel; + uint8_t dip_vld; + uint8_t wnd_mode_sel; +}; + +struct udma_dip { + uint8_t dgid[UDMA_GID_SIZE]; + uint32_t dip_idx; + struct list_head node; /* all dips are on a list */ +}; + +#define UDMA_CONGEST_SIZE 64 +#define UDMA_SCC_DIP_INVALID_IDX (-1) + +enum { + CONGEST_DCQCN, + CONGEST_LDCP, + CONGEST_HC3, + CONGEST_DIP, +}; + +enum { + DCQCN_ALG, + WINDOW_ALG, +}; + +enum { + UNSUPPORT_CONGEST_DEGREE, + SUPPORT_CONGEST_DEGREE, +}; + +enum { + DIP_INVALID, + DIP_VALID, +}; + +enum { + WND_LIMIT, + WND_UNLIMIT, +}; + +#define gen_qpn(high, mid, low) ((high) | (mid) | (low)) + +int udma_modify_qp_common(struct udma_qp *qp, + const struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask ubcore_mask, + enum udma_qp_state curr_state, + enum udma_qp_state new_state); +int udma_fill_qp_attr(struct udma_dev *udma_dev, struct udma_qp_attr *qp_attr, + const struct ubcore_tp_cfg *cfg, struct ubcore_udata *udata); +int udma_create_qp_common(struct udma_dev *udma_dev, struct udma_qp *qp, + struct ubcore_udata *udata); +void udma_destroy_qp_common(struct udma_dev *udma_dev, struct udma_qp *qp); +void init_jetty_x_qpn_bitmap(struct udma_dev *dev, + struct udma_qpn_bitmap *qpn_map, + uint32_t jetty_x_shift, uint32_t prefix, + uint32_t jid); +void clean_jetty_x_qpn_bitmap(struct udma_qpn_bitmap *qpn_map); +int udma_flush_cqe(struct udma_dev *udma_dev, struct udma_qp *udma_qp, + uint32_t sq_pi); +void udma_qp_event(struct udma_dev *udma_dev, uint32_t qpn, int event_type); +void copy_send_jfc(struct udma_qp *from_qp, struct udma_qp *to_qp); +int udma_set_dca_buf(struct udma_dev *dev, struct udma_qp *qp); + +#endif /* _UDMA_QP_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_segment.c b/drivers/ub/hw/hns3/hns3_udma_segment.c new file mode 100644 index 0000000000000000000000000000000000000000..817bea1a83fd87bd71fac0458feb1552caa3ac2b --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_segment.c @@ -0,0 +1,409 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include "urma/ubcore_types.h" +#include "hns3_udma_abi.h" +#include "hns3_udma_hem.h" +#include "hns3_udma_cmd.h" +#include "hns3_udma_dfx.h" +#include "hns3_udma_segment.h" + +static uint32_t hw_index_to_key(int ind) +{ + return ((uint32_t)ind << SEG_KEY_OFFSET); +} + +uint64_t key_to_hw_index(uint32_t key) +{ + return (key >> SEG_KEY_OFFSET); +} + +static int udma_hw_create_mpt(struct udma_dev *udma_dev, + struct udma_cmd_mailbox *mailbox, + uint64_t mpt_index) +{ + struct udma_cmq_desc desc; + struct udma_mbox *mb; + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mbox_desc_init(mb, mailbox->dma, 0, mpt_index, UDMA_CMD_CREATE_MPT); + + return udma_cmd_mbox(udma_dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); +} + +int udma_hw_destroy_mpt(struct udma_dev *udma_dev, + struct udma_cmd_mailbox *mailbox, + uint64_t mpt_index) +{ + struct udma_cmq_desc desc; + struct udma_mbox *mb; + + mb = (struct udma_mbox *)desc.data; + udma_cmq_setup_basic_desc(&desc, UDMA_OPC_POST_MB, false); + mbox_desc_init(mb, 0, 0, mpt_index, UDMA_CMD_DESTROY_MPT); + + return udma_cmd_mbox(udma_dev, &desc, UDMA_CMD_TIMEOUT_MSECS, 0); +} + +static int alloc_seg_key(struct udma_dev *udma_dev, struct udma_seg *seg) +{ + struct udma_ida *seg_ida = &udma_dev->seg_table.seg_ida; + int err; + int id; + + id = ida_alloc_range(&seg_ida->ida, seg_ida->min, seg_ida->max, + GFP_KERNEL); + if (id < 0) { + dev_err(udma_dev->dev, "failed to alloc id for MR key, id(%d)\n", + id); + return -ENOMEM; + } + + seg->key = hw_index_to_key(id); + + err = udma_table_get(udma_dev, &udma_dev->seg_table.table, + (uint64_t)id); + if (err) { + dev_err(udma_dev->dev, + "failed to alloc mtpt, ret = %d.\n", err); + goto err_free_bitmap; + } + + return 0; +err_free_bitmap: + ida_free(&seg_ida->ida, id); + return err; +} + +static int alloc_seg_pbl(struct udma_dev *udma_dev, struct udma_seg *seg, + bool is_user) +{ + struct udma_buf_attr buf_attr = {}; + int err; + + seg->pbl_hop_num = udma_dev->caps.pbl_hop_num; + buf_attr.page_shift = udma_dev->caps.pbl_buf_pg_sz + PAGE_SHIFT; + buf_attr.region[0].size = seg->size; + buf_attr.region[0].hopnum = seg->pbl_hop_num; + buf_attr.region_count = 1; + buf_attr.mtt_only = false; + + err = udma_mtr_create(udma_dev, &seg->pbl_mtr, &buf_attr, + udma_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT, + seg->iova, is_user); + if (err) + dev_err(udma_dev->dev, "failed to alloc pbl mtr, ret = %d.\n", + err); + else + seg->npages = seg->pbl_mtr.hem_cfg.buf_pg_count; + + return err; +} + +static int set_mtpt_pbl(struct udma_dev *udma_dev, + struct udma_mpt_entry *mpt_entry, + struct udma_seg *seg) +{ + uint64_t pages[UDMA_MAX_INNER_MTPT_NUM] = {}; + uint64_t pbl_ba; + int i, count; + + count = udma_mtr_find(udma_dev, &seg->pbl_mtr, 0, pages, + min_t(int, ARRAY_SIZE(pages), seg->npages), + &pbl_ba); + if (count < 1) { + dev_err(udma_dev->dev, "failed to find PBL mtr, count = %d.\n", + count); + return -ENOBUFS; + } + + /* Aligned to the hardware address access unit */ + for (i = 0; i < count; i++) + pages[i] >>= PA_PAGE_SHIFT; + + mpt_entry->pbl_size = cpu_to_le32(seg->npages); + mpt_entry->pbl_ba_l = cpu_to_le32(pbl_ba >> MPT_PAGE_OFFSET); + udma_reg_write(mpt_entry, MPT_PBL_BA_H, + upper_32_bits(pbl_ba >> MPT_PAGE_OFFSET)); + mpt_entry->pa0_l = cpu_to_le32(lower_32_bits(pages[0])); + udma_reg_write(mpt_entry, MPT_PA0_H, upper_32_bits(pages[0])); + mpt_entry->pa1_l = cpu_to_le32(lower_32_bits(pages[1])); + udma_reg_write(mpt_entry, MPT_PA1_H, upper_32_bits(pages[1])); + udma_reg_write(mpt_entry, MPT_PBL_BUF_PG_SZ, + to_hr_hw_page_shift(seg->pbl_mtr.hem_cfg.buf_pg_shift)); + + return 0; +} + +static int udma_write_seg_mpt(struct udma_dev *udma_dev, + void *mb_buf, struct udma_seg *seg) +{ + struct udma_mpt_entry *mpt_entry; + int ret = 0; + + mpt_entry = (struct udma_mpt_entry *)mb_buf; + memset(mpt_entry, 0, sizeof(*mpt_entry)); + + udma_reg_write(mpt_entry, MPT_ST, MPT_ST_VALID); + udma_reg_write(mpt_entry, MPT_PD, seg->pd); + udma_reg_enable(mpt_entry, MPT_L_INV_EN); + + udma_reg_write(mpt_entry, MPT_RW_EN, + !!(seg->access & UBCORE_ACCESS_REMOTE_WRITE)); + udma_reg_write(mpt_entry, MPT_LW_EN, + !!(seg->access & UBCORE_ACCESS_LOCAL_WRITE)); + udma_reg_write(mpt_entry, MPT_R_INV_EN, + !!(seg->access & UBCORE_ACCESS_REMOTE_INVALIDATE)); + + mpt_entry->len_l = cpu_to_le32(lower_32_bits(seg->size)); + mpt_entry->len_h = cpu_to_le32(upper_32_bits(seg->size)); + mpt_entry->lkey = cpu_to_le32(seg->key); + mpt_entry->va_l = cpu_to_le32(lower_32_bits(seg->iova)); + mpt_entry->va_h = cpu_to_le32(upper_32_bits(seg->iova)); + + udma_reg_write(mpt_entry, MPT_PERSIST_EN, 1); + + if (seg->pbl_hop_num != UDMA_HOP_NUM_0) + udma_reg_write(mpt_entry, MPT_PBL_HOP_NUM, seg->pbl_hop_num); + + udma_reg_write(mpt_entry, MPT_PBL_BA_PG_SZ, + to_hr_hw_page_shift(seg->pbl_mtr.hem_cfg.ba_pg_shift)); + udma_reg_enable(mpt_entry, MPT_INNER_PA_VLD); + + ret = set_mtpt_pbl(udma_dev, mpt_entry, seg); + + return ret; +} + +static int udma_seg_enable(struct udma_dev *udma_dev, struct udma_seg *seg) +{ + uint64_t seg_idx = key_to_hw_index(seg->key); + struct device *dev = udma_dev->dev; + struct udma_cmd_mailbox *mailbox; + int ret; + + /* Allocate mailbox memory */ + mailbox = udma_alloc_cmd_mailbox(udma_dev); + if (IS_ERR(mailbox)) { + ret = PTR_ERR(mailbox); + return ret; + } + + ret = udma_write_seg_mpt(udma_dev, mailbox->buf, seg); + if (ret) { + dev_err(dev, "failed to write mtpt, ret = %d.\n", ret); + goto err_page; + } + + ret = udma_hw_create_mpt(udma_dev, mailbox, + seg_idx & (udma_dev->caps.num_mtpts - 1)); + if (ret) { + dev_err(dev, "failed to create mpt, ret = %d.\n", ret); + goto err_page; + } + +err_page: + udma_free_cmd_mailbox(udma_dev, mailbox); + + return ret; +} + +static void free_seg_pbl(struct udma_dev *udma_dev, struct udma_seg *seg) +{ + udma_mtr_destroy(udma_dev, &seg->pbl_mtr); +} + +static void free_seg_key(struct udma_dev *udma_dev, struct udma_seg *seg) +{ + uint64_t obj = key_to_hw_index(seg->key); + + udma_table_put(udma_dev, &udma_dev->seg_table.table, obj); + ida_free(&udma_dev->seg_table.seg_ida.ida, (int)obj); +} + +static void store_seg_id(struct udma_dev *udma_dev, struct udma_seg *seg) +{ + struct seg_list *seg_new; + struct seg_list *seg_now; + unsigned long flags; + spinlock_t *lock; + int ret; + int i; + + ret = udma_find_dfx_dev(udma_dev, &i); + if (ret) + return; + + seg_new = kzalloc(sizeof(struct seg_list), GFP_KERNEL); + if (seg_new == NULL) + return; + + lock = &g_udma_dfx_list[i].dfx->seg_list->node_lock; + spin_lock_irqsave(lock, flags); + list_for_each_entry(seg_now, + &g_udma_dfx_list[i].dfx->seg_list->node, node) { + if (seg_now->key_id == seg->key) { + seg_now->pd = seg->pd; + seg_now->iova = seg->iova; + seg_now->len = seg->size; + goto found; + } + } + + seg_new->pd = seg->pd; + seg_new->iova = seg->iova; + seg_new->len = seg->size; + seg_new->key_id = seg->key; + list_add(&seg_new->node, &g_udma_dfx_list[i].dfx->seg_list->node); + spin_unlock_irqrestore(lock, flags); + + return; + +found: + spin_unlock_irqrestore(lock, flags); + kfree(seg_new); +} + +static void delete_seg_id(struct udma_dev *udma_dev, struct udma_seg *seg) +{ + struct seg_list *seg_now, *seg_tmp; + unsigned long flags; + spinlock_t *lock; + int ret; + int i; + + ret = udma_find_dfx_dev(udma_dev, &i); + if (ret) + return; + + lock = &g_udma_dfx_list[i].dfx->seg_list->node_lock; + spin_lock_irqsave(lock, flags); + list_for_each_entry_safe(seg_now, seg_tmp, + &g_udma_dfx_list[i].dfx->seg_list->node, + node) { + if (seg_now->key_id == seg->key) { + list_del(&seg_now->node); + kfree(seg_now); + spin_unlock_irqrestore(lock, flags); + return; + } + } + spin_unlock_irqrestore(lock, flags); +} + +struct ubcore_target_seg *udma_register_seg(struct ubcore_device *dev, + const struct ubcore_seg_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + struct udma_ucontext *udma_ctx; + struct udma_seg *seg; + int ret; + + if (cfg->flag.bs.access >= URMA_SEG_ACCESS_GUARD) { + dev_err(udma_dev->dev, "Invalid segment access 0x%x.\n", + cfg->flag.bs.access); + return NULL; + } + + seg = kcalloc(1, sizeof(*seg), GFP_KERNEL); + if (!seg) + return NULL; + + udma_ctx = to_udma_ucontext(udata->uctx); + seg->iova = cfg->va; + seg->size = cfg->len; + seg->pd = udma_ctx->pdn; + seg->access = cfg->flag.bs.access; + + ret = alloc_seg_key(udma_dev, seg); + if (ret) + goto err_alloc_key; + + ret = alloc_seg_pbl(udma_dev, seg, !!udata); + if (ret) + goto err_alloc_pbl; + + ret = udma_seg_enable(udma_dev, seg); + if (ret) + goto err_enable_seg; + seg->enabled = 1; + seg->ubcore_seg.seg.key_id = seg->key; + + if (dfx_switch) + store_seg_id(udma_dev, seg); + + return &seg->ubcore_seg; + +err_enable_seg: + free_seg_pbl(udma_dev, seg); +err_alloc_pbl: + free_seg_key(udma_dev, seg); +err_alloc_key: + kfree(seg); + return NULL; +} + +void udma_seg_free(struct udma_dev *udma_dev, struct udma_seg *seg) +{ + int ret; + + if (seg->enabled) { + ret = udma_hw_destroy_mpt(udma_dev, NULL, + key_to_hw_index(seg->key) & + (udma_dev->caps.num_mtpts - 1)); + if (ret) + dev_warn(udma_dev->dev, "failed to destroy mpt, ret = %d.\n", + ret); + } + + free_seg_pbl(udma_dev, seg); + free_seg_key(udma_dev, seg); +} + +int udma_unregister_seg(struct ubcore_target_seg *seg) +{ + struct udma_dev *udma_dev = to_udma_dev(seg->ub_dev); + struct udma_seg *udma_seg = to_udma_seg(seg); + + if (dfx_switch) + delete_seg_id(udma_dev, udma_seg); + + udma_seg_free(udma_dev, udma_seg); + kfree(udma_seg); + + return 0; +} + +struct ubcore_target_seg *udma_import_seg(struct ubcore_device *dev, + const struct ubcore_target_seg_cfg *cfg, + struct ubcore_udata *udata) +{ + struct ubcore_target_seg *tseg; + + tseg = kcalloc(1, sizeof(*tseg), GFP_KERNEL); + if (!tseg) + return NULL; + + return tseg; +} + +int udma_unimport_seg(struct ubcore_target_seg *tseg) +{ + kfree(tseg); + + return 0; +} diff --git a/drivers/ub/hw/hns3/hns3_udma_segment.h b/drivers/ub/hw/hns3/hns3_udma_segment.h new file mode 100644 index 0000000000000000000000000000000000000000..c97ab98b57dd370975fabc13f0d8c17f3630dc51 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_segment.h @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_SEGMENT_H +#define _UDMA_SEGMENT_H + +#include +#include +#include +#include "urma/ubcore_opcode.h" +#include "hns3_udma_device.h" +#include "hns3_udma_common.h" + +#define SEG_KEY_OFFSET 8 +#define MPT_PAGE_OFFSET 3 +#define PA_PAGE_SHIFT 6 +#define MPT_VA_H_SHIFT 32 +#define MPT_LEN_H_SHIFT 32 + +enum { + MPT_ST_VALID = 0x1, +}; + +struct udma_mpt_entry { + uint32_t mpt_context1[4]; + uint32_t len_l; + uint32_t len_h; + uint32_t lkey; + uint32_t va_l; + uint32_t va_h; + uint32_t pbl_size; + uint32_t pbl_ba_l; + uint32_t mpt_context2; + uint32_t pa0_l; + uint32_t mpt_context3; + uint32_t pa1_l; + uint32_t mpt_context4; +}; + +#define MPT_FIELD_LOC(h, l) ((uint64_t)(h) << 32 | (l)) + +#define MPT_ST MPT_FIELD_LOC(1, 0) +#define MPT_PBL_HOP_NUM MPT_FIELD_LOC(3, 2) +#define MPT_PBL_BA_PG_SZ MPT_FIELD_LOC(7, 4) +#define MPT_PD MPT_FIELD_LOC(31, 8) +#define MPT_R_INV_EN MPT_FIELD_LOC(33, 33) +#define MPT_L_INV_EN MPT_FIELD_LOC(34, 34) +#define MPT_RW_EN MPT_FIELD_LOC(38, 38) +#define MPT_LW_EN MPT_FIELD_LOC(39, 39) +#define MPT_PA MPT_FIELD_LOC(65, 65) +#define MPT_INNER_PA_VLD MPT_FIELD_LOC(71, 71) +#define MPT_LEN_L MPT_FIELD_LOC(159, 128) +#define MPT_LEN_H MPT_FIELD_LOC(191, 160) +#define MPT_LKEY MPT_FIELD_LOC(223, 192) +#define MPT_VA_L MPT_FIELD_LOC(255, 224) +#define MPT_VA_H MPT_FIELD_LOC(287, 256) +#define MPT_PBL_BA_H MPT_FIELD_LOC(380, 352) +#define MPT_PA0_H MPT_FIELD_LOC(441, 416) +#define MPT_PA1_L MPT_FIELD_LOC(579, 448) +#define MPT_PA1_H MPT_FIELD_LOC(505, 480) +#define MPT_PERSIST_EN MPT_FIELD_LOC(506, 506) +#define MPT_PBL_BUF_PG_SZ MPT_FIELD_LOC(511, 508) + +#define UDMA_MAX_INNER_MTPT_NUM 2 + +struct ubcore_target_seg *udma_register_seg(struct ubcore_device *dev, + const struct ubcore_seg_cfg *cfg, + struct ubcore_udata *udata); +int udma_unregister_seg(struct ubcore_target_seg *seg); +struct ubcore_target_seg *udma_import_seg(struct ubcore_device *dev, + const struct ubcore_target_seg_cfg *cfg, + struct ubcore_udata *udata); +int udma_unimport_seg(struct ubcore_target_seg *tseg); +uint64_t key_to_hw_index(uint32_t key); + +#endif /* _UDMA_SEGMENT_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_sysfs.c b/drivers/ub/hw/hns3/hns3_udma_sysfs.c new file mode 100644 index 0000000000000000000000000000000000000000..0d4e3ac373e7d78486c0fc02413204cd411c7075 --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_sysfs.c @@ -0,0 +1,543 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include +#include "hns3_udma_cmd.h" +#include "hns3_udma_sysfs.h" + +static enum udma_opcode_type scc_opcode[] = { + UDMA_OPC_CFG_DCQCN_PARAM, + UDMA_OPC_CFG_LDCP_PARAM, + UDMA_OPC_CFG_HC3_PARAM, + UDMA_OPC_CFG_DIP_PARAM, +}; + +static int udma_config_scc_param(struct udma_dev *udma_dev, + uint8_t port_num, enum udma_cong_type algo) +{ + struct udma_scc_param *scc_param; + struct udma_cmq_desc desc; + struct udma_port *pdata; + int ret; + + if (port_num >= udma_dev->caps.num_ports) { + dev_err_ratelimited(udma_dev->dev, + "invalid port num %u.\n", port_num); + return -ENODEV; + } + + if (algo >= UDMA_CONG_TYPE_TOTAL) { + dev_err_ratelimited(udma_dev->dev, "invalid SCC algo.\n"); + return -EINVAL; + } + + udma_cmq_setup_basic_desc(&desc, scc_opcode[algo], false); + pdata = &udma_dev->port_data[port_num]; + scc_param = &pdata->scc_param[algo]; + if (!scc_param) { + dev_err_ratelimited(udma_dev->dev, "scc_param has been freed.\n"); + return -ENODEV; + } + + memcpy(&desc.data, scc_param, sizeof(scc_param->param)); + + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) + dev_err_ratelimited(udma_dev->dev, + "failed to configure scc param, opcode: 0x%x, ret = %d.\n", + le16_to_cpu(desc.opcode), ret); + + return ret; +} + +static int udma_query_scc_param(struct udma_dev *udma_dev, + uint8_t port_num, enum udma_cong_type algo) +{ + struct udma_scc_param *scc_param; + struct udma_cmq_desc desc; + struct udma_port *pdata; + int ret; + + if (port_num >= udma_dev->caps.num_ports) { + dev_err_ratelimited(udma_dev->dev, + "invalid port num %u.\n", port_num); + return -ENODEV; + } + + if (algo >= UDMA_CONG_TYPE_TOTAL) { + dev_err_ratelimited(udma_dev->dev, "invalid SCC algo.\n"); + return -EINVAL; + } + + udma_cmq_setup_basic_desc(&desc, scc_opcode[algo], true); + ret = udma_cmq_send(udma_dev, &desc, 1); + if (ret) { + dev_err_ratelimited(udma_dev->dev, + "failed to query scc param, opecode: 0x%x, ret = %d.\n", + le16_to_cpu(desc.opcode), ret); + return ret; + } + + pdata = &udma_dev->port_data[port_num]; + scc_param = &pdata->scc_param[algo]; + memcpy(scc_param, &desc.data, sizeof(scc_param->param)); + + return 0; +} + +static void scc_param_config_work(struct work_struct *work) +{ + struct udma_scc_param *scc_param = container_of(work, + struct udma_scc_param, + scc_cfg_dwork.work); + struct udma_dev *udma_dev = scc_param->udma_dev; + + udma_config_scc_param(udma_dev, scc_param->port_num, + scc_param->algo_type); +} + +static int alloc_scc_param(struct udma_dev *udma_dev, + struct udma_port *pdata) +{ + struct udma_scc_param *scc_param; + int i; + + scc_param = kcalloc(UDMA_CONG_TYPE_TOTAL, sizeof(*scc_param), + GFP_KERNEL); + if (!scc_param) + return -ENOMEM; + + for (i = 0; i < UDMA_CONG_TYPE_TOTAL; i++) { + scc_param[i].algo_type = (enum udma_cong_type)i; + scc_param[i].timestamp = jiffies; + scc_param[i].udma_dev = udma_dev; + scc_param[i].port_num = pdata->port_num; + scc_param[i].configured = false; + INIT_DELAYED_WORK(&scc_param[i].scc_cfg_dwork, + scc_param_config_work); + } + + pdata->scc_param = scc_param; + + return 0; +} + +static int scc_attr_check(struct udma_port_cc_attr *scc_attr) +{ + if (WARN_ON(scc_attr->size > sizeof(uint32_t))) + return -EINVAL; + + if (WARN_ON(scc_attr->algo_type >= UDMA_CONG_TYPE_TOTAL)) + return -EINVAL; + + return 0; +} + +static ssize_t scc_attr_show(struct udma_port *pdata, + struct udma_port_attribute *attr, char *buf) +{ + struct udma_port_cc_attr *scc_attr; + struct udma_scc_param *scc_param; + unsigned long exp_time; + uint32_t val = 0; + int ret; + + scc_attr = container_of(attr, struct udma_port_cc_attr, port_attr); + ret = scc_attr_check(scc_attr); + if (ret) + return ret; + + scc_param = &pdata->scc_param[scc_attr->algo_type]; + + /* Only HW param need be queried */ + if (scc_attr->offset < offsetof(typeof(*scc_param), lifespan)) { + exp_time = scc_param->timestamp + + msecs_to_jiffies(scc_param->lifespan); + if (time_is_before_eq_jiffies(exp_time)) { + scc_param->timestamp = jiffies; + udma_query_scc_param(pdata->udma_dev, + pdata->port_num, + scc_attr->algo_type); + } + } + + memcpy(&val, (void *)scc_param + scc_attr->offset, scc_attr->size); + + return sysfs_emit(buf, "%u\n", le32_to_cpu(val)); +} + +static ssize_t scc_attr_store(struct udma_port *pdata, + struct udma_port_attribute *attr, + const char *buf, size_t count) +{ + struct udma_port_cc_attr *scc_attr; + struct udma_scc_param *scc_param; + uint64_t lifespan_jiffies; + unsigned long exp_time; + uint32_t attr_val; + uint32_t val; + int ret; + + scc_attr = container_of(attr, struct udma_port_cc_attr, port_attr); + ret = scc_attr_check(scc_attr); + if (ret) + return ret; + + if (kstrtou32(buf, 0, &val)) + return -EINVAL; + + if (val > scc_attr->max || val < scc_attr->min) + return -EINVAL; + + attr_val = cpu_to_le32(val); + scc_param = &pdata->scc_param[scc_attr->algo_type]; + + /* get current params of this scc algo before configure it first time */ + if (scc_param->configured == false) + udma_query_scc_param(pdata->udma_dev, scc_param->port_num, + scc_param->algo_type); + + memcpy((void *)scc_param + scc_attr->offset, &attr_val, scc_attr->size); + + /* lifespan is only used for driver */ + if (scc_attr->offset >= offsetof(typeof(*scc_param), lifespan)) + return count; + + lifespan_jiffies = msecs_to_jiffies(scc_param->lifespan); + exp_time = scc_param->timestamp + lifespan_jiffies; + + if (time_is_before_eq_jiffies(exp_time)) { + scc_param->timestamp = jiffies; + queue_delayed_work(pdata->udma_dev->irq_workq, + &scc_param->scc_cfg_dwork, lifespan_jiffies); + scc_param->configured = true; + } + + return count; +} + +static umode_t scc_attr_is_visible(struct kobject *kobj, + struct attribute *attr, int i) +{ + struct udma_port_attribute *port_attr; + struct udma_port_cc_attr *scc_attr; + struct udma_dev *udma_dev; + struct udma_port *pdata; + + port_attr = container_of(attr, struct udma_port_attribute, attr); + scc_attr = container_of(port_attr, struct udma_port_cc_attr, port_attr); + pdata = container_of(kobj, struct udma_port, kobj); + udma_dev = pdata->udma_dev; + + if (!(udma_dev->caps.flags & UDMA_CAP_FLAG_QP_FLOW_CTRL)) + return 0; + + if (!(udma_dev->caps.cong_type & (1 << scc_attr->algo_type))) + return 0; + + return ATTR_RW_RONLY_RONLY; +} + +#define __UDMA_SCC_ATTR(_name, _type, _offset, _size, _min, _max) { \ + .port_attr = __ATTR(_name, 0644, scc_attr_show, scc_attr_store), \ + .algo_type = (_type), \ + .offset = (_offset), \ + .size = (_size), \ + .min = (_min), \ + .max = (_max), \ +} + +#define UDMA_DCQCN_CC_ATTR_RW(_name, NAME) \ + struct udma_port_cc_attr udma_port_attr_dcqcn_##_name = \ + __UDMA_SCC_ATTR(_name, UDMA_CONG_TYPE_DCQCN, \ + UDMA_DCQCN_##NAME##_OFS, \ + UDMA_DCQCN_##NAME##_SZ, \ + 0, UDMA_DCQCN_##NAME##_MAX) + +UDMA_DCQCN_CC_ATTR_RW(ai, AI); +UDMA_DCQCN_CC_ATTR_RW(f, F); +UDMA_DCQCN_CC_ATTR_RW(tkp, TKP); +UDMA_DCQCN_CC_ATTR_RW(tmp, TMP); +UDMA_DCQCN_CC_ATTR_RW(alp, ALP); +UDMA_DCQCN_CC_ATTR_RW(max_speed, MAX_SPEED); +UDMA_DCQCN_CC_ATTR_RW(g, G); +UDMA_DCQCN_CC_ATTR_RW(al, AL); +UDMA_DCQCN_CC_ATTR_RW(cnp_time, CNP_TIME); +UDMA_DCQCN_CC_ATTR_RW(ashift, ASHIFT); +UDMA_DCQCN_CC_ATTR_RW(lifespan, LIFESPAN); + +static struct attribute *dcqcn_param_attrs[] = { + &udma_port_attr_dcqcn_ai.port_attr.attr, + &udma_port_attr_dcqcn_f.port_attr.attr, + &udma_port_attr_dcqcn_tkp.port_attr.attr, + &udma_port_attr_dcqcn_tmp.port_attr.attr, + &udma_port_attr_dcqcn_alp.port_attr.attr, + &udma_port_attr_dcqcn_max_speed.port_attr.attr, + &udma_port_attr_dcqcn_g.port_attr.attr, + &udma_port_attr_dcqcn_al.port_attr.attr, + &udma_port_attr_dcqcn_cnp_time.port_attr.attr, + &udma_port_attr_dcqcn_ashift.port_attr.attr, + &udma_port_attr_dcqcn_lifespan.port_attr.attr, + NULL, +}; + +static const struct attribute_group dcqcn_cc_param_group = { + .name = "dcqcn_cc_param", + .attrs = dcqcn_param_attrs, + .is_visible = scc_attr_is_visible, +}; + +#define UDMA_LDCP_CC_ATTR_RW(_name, NAME) \ + struct udma_port_cc_attr udma_port_attr_ldcp_##_name = \ + __UDMA_SCC_ATTR(_name, UDMA_CONG_TYPE_LDCP, \ + UDMA_LDCP_##NAME##_OFS, \ + UDMA_LDCP_##NAME##_SZ, \ + 0, UDMA_LDCP_##NAME##_MAX) + +UDMA_LDCP_CC_ATTR_RW(cwd0, CWD0); +UDMA_LDCP_CC_ATTR_RW(alpha, ALPHA); +UDMA_LDCP_CC_ATTR_RW(gamma, GAMMA); +UDMA_LDCP_CC_ATTR_RW(beta, BETA); +UDMA_LDCP_CC_ATTR_RW(eta, ETA); +UDMA_LDCP_CC_ATTR_RW(lifespan, LIFESPAN); + +static struct attribute *ldcp_param_attrs[] = { + &udma_port_attr_ldcp_cwd0.port_attr.attr, + &udma_port_attr_ldcp_alpha.port_attr.attr, + &udma_port_attr_ldcp_gamma.port_attr.attr, + &udma_port_attr_ldcp_beta.port_attr.attr, + &udma_port_attr_ldcp_eta.port_attr.attr, + &udma_port_attr_ldcp_lifespan.port_attr.attr, + NULL, +}; + +static const struct attribute_group ldcp_cc_param_group = { + .name = "ldcp_cc_param", + .attrs = ldcp_param_attrs, + .is_visible = scc_attr_is_visible, +}; + +#define UDMA_HC3_CC_ATTR_RW(_name, NAME) \ + struct udma_port_cc_attr udma_port_attr_hc3_##_name = \ + __UDMA_SCC_ATTR(_name, UDMA_CONG_TYPE_HC3, \ + UDMA_HC3_##NAME##_OFS, \ + UDMA_HC3_##NAME##_SZ, \ + 0, UDMA_HC3_##NAME##_MAX) + +UDMA_HC3_CC_ATTR_RW(initial_window, INITIAL_WINDOW); +UDMA_HC3_CC_ATTR_RW(bandwidth, BANDWIDTH); +UDMA_HC3_CC_ATTR_RW(qlen_shift, QLEN_SHIFT); +UDMA_HC3_CC_ATTR_RW(port_usage_shift, PORT_USAGE_SHIFT); +UDMA_HC3_CC_ATTR_RW(over_period, OVER_PERIOD); +UDMA_HC3_CC_ATTR_RW(max_stage, MAX_STAGE); +UDMA_HC3_CC_ATTR_RW(gamma_shift, GAMMA_SHIFT); +UDMA_HC3_CC_ATTR_RW(lifespan, LIFESPAN); + +static struct attribute *hc3_param_attrs[] = { + &udma_port_attr_hc3_initial_window.port_attr.attr, + &udma_port_attr_hc3_bandwidth.port_attr.attr, + &udma_port_attr_hc3_qlen_shift.port_attr.attr, + &udma_port_attr_hc3_port_usage_shift.port_attr.attr, + &udma_port_attr_hc3_over_period.port_attr.attr, + &udma_port_attr_hc3_max_stage.port_attr.attr, + &udma_port_attr_hc3_gamma_shift.port_attr.attr, + &udma_port_attr_hc3_lifespan.port_attr.attr, + NULL, +}; + +static const struct attribute_group hc3_cc_param_group = { + .name = "hc3_cc_param", + .attrs = hc3_param_attrs, + .is_visible = scc_attr_is_visible, +}; + +#define UDMA_DIP_CC_ATTR_RW(_name, NAME) \ + struct udma_port_cc_attr udma_port_attr_dip_##_name = \ + __UDMA_SCC_ATTR(_name, UDMA_CONG_TYPE_DIP, \ + UDMA_DIP_##NAME##_OFS, \ + UDMA_DIP_##NAME##_SZ, \ + 0, UDMA_DIP_##NAME##_MAX) + +UDMA_DIP_CC_ATTR_RW(ai, AI); +UDMA_DIP_CC_ATTR_RW(f, F); +UDMA_DIP_CC_ATTR_RW(tkp, TKP); +UDMA_DIP_CC_ATTR_RW(tmp, TMP); +UDMA_DIP_CC_ATTR_RW(alp, ALP); +UDMA_DIP_CC_ATTR_RW(max_speed, MAX_SPEED); +UDMA_DIP_CC_ATTR_RW(g, G); +UDMA_DIP_CC_ATTR_RW(al, AL); +UDMA_DIP_CC_ATTR_RW(cnp_time, CNP_TIME); +UDMA_DIP_CC_ATTR_RW(ashift, ASHIFT); +UDMA_DIP_CC_ATTR_RW(lifespan, LIFESPAN); + +static struct attribute *dip_param_attrs[] = { + &udma_port_attr_dip_ai.port_attr.attr, + &udma_port_attr_dip_f.port_attr.attr, + &udma_port_attr_dip_tkp.port_attr.attr, + &udma_port_attr_dip_tmp.port_attr.attr, + &udma_port_attr_dip_alp.port_attr.attr, + &udma_port_attr_dip_max_speed.port_attr.attr, + &udma_port_attr_dip_g.port_attr.attr, + &udma_port_attr_dip_al.port_attr.attr, + &udma_port_attr_dip_cnp_time.port_attr.attr, + &udma_port_attr_dip_ashift.port_attr.attr, + &udma_port_attr_dip_lifespan.port_attr.attr, + NULL, +}; + +static const struct attribute_group dip_cc_param_group = { + .name = "dip_cc_param", + .attrs = dip_param_attrs, + .is_visible = scc_attr_is_visible, +}; + +const struct attribute_group *udma_attr_port_groups[] = { + &dcqcn_cc_param_group, + &ldcp_cc_param_group, + &hc3_cc_param_group, + &dip_cc_param_group, + NULL, +}; + +static ssize_t udma_port_attr_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct udma_port_attribute *port_attr; + struct udma_port *port; + + port_attr = container_of(attr, struct udma_port_attribute, attr); + port = container_of(kobj, struct udma_port, kobj); + if (!port_attr->show) + return -EIO; + + return port_attr->show(port, port_attr, buf); +} + +static ssize_t udma_port_attr_store(struct kobject *kobj, + struct attribute *attr, + const char *buf, size_t count) +{ + struct udma_port_attribute *port_attr; + struct udma_port *p; + + port_attr = container_of(attr, struct udma_port_attribute, attr); + p = container_of(kobj, struct udma_port, kobj); + if (!port_attr->store) + return -EIO; + + return port_attr->store(p, port_attr, buf, count); +} + +static void udma_port_release(struct kobject *kobj) +{ + struct udma_port *pdata = container_of(kobj, struct udma_port, kobj); + + kfree(pdata->scc_param); + pdata->scc_param = NULL; +} + +static const struct sysfs_ops udma_port_ops = { + .show = udma_port_attr_show, + .store = udma_port_attr_store, +}; + +static struct kobj_type udma_port_ktype = { + .release = udma_port_release, + .sysfs_ops = &udma_port_ops, +}; + +static int udma_register_port_sysfs(struct udma_dev *udma_dev, uint8_t port_num, + struct kobject *kobj) +{ + struct udma_port *pdata; + int ret; + + if (port_num >= udma_dev->caps.num_ports) { + dev_err(udma_dev->dev, "fail to create port sysfs for invalid port %u.\n", + port_num); + return -ENODEV; + } + + pdata = &udma_dev->port_data[port_num]; + pdata->udma_dev = udma_dev; + pdata->port_num = port_num; + ret = kobject_init_and_add(&pdata->kobj, &udma_port_ktype, + kobj, "cc_param"); + if (ret) { + dev_err(udma_dev->dev, "fail to create port(%u) sysfs, ret = %d.\n", + port_num, ret); + goto fail_kobj; + } + kobject_uevent(&pdata->kobj, KOBJ_ADD); + + ret = sysfs_create_groups(&pdata->kobj, udma_attr_port_groups); + if (ret) { + dev_err(udma_dev->dev, + "fail to create port(%u) cc param sysfs, ret = %d.\n", + port_num, ret); + goto fail_kobj; + } + + ret = alloc_scc_param(udma_dev, pdata); + if (ret) { + dev_err(udma_dev->dev, "alloc scc param failed, ret = %d!\n", + ret); + goto fail_group; + } + + return ret; + +fail_group: + sysfs_remove_groups(&pdata->kobj, udma_attr_port_groups); +fail_kobj: + kobject_put(&pdata->kobj); + return ret; +} + +static void udma_unregister_port_sysfs(struct udma_dev *udma_dev, uint8_t port_num) +{ + struct udma_port *pdata; + + pdata = &udma_dev->port_data[port_num]; + sysfs_remove_groups(&pdata->kobj, udma_attr_port_groups); + kobject_put(&pdata->kobj); +} + +int udma_register_cc_sysfs(struct udma_dev *udma_dev) +{ + uint8_t i; + uint8_t j; + + for (i = 0; i < udma_dev->caps.num_ports; i++) { + if (udma_register_port_sysfs(udma_dev, i, &udma_dev->dev->kobj) != 0) + goto err_port_attr; + } + + return 0; + +err_port_attr: + for (j = 0; j < i; j++) + udma_unregister_port_sysfs(udma_dev, j); + + return -EPERM; +} + +void udma_unregister_cc_sysfs(struct udma_dev *udma_dev) +{ + int i; + + for (i = 0; i < udma_dev->caps.num_ports; i++) + udma_unregister_port_sysfs(udma_dev, i); +} diff --git a/drivers/ub/hw/hns3/hns3_udma_sysfs.h b/drivers/ub/hw/hns3/hns3_udma_sysfs.h new file mode 100644 index 0000000000000000000000000000000000000000..54c3b1c5ebfd52cd9a4edcb1a197e32bb53f57df --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_sysfs.h @@ -0,0 +1,173 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_SYSFS_H +#define _UDMA_SYSFS_H + +#include +#include "hns3_udma_device.h" + +#define UDMA_DCQCN_AI_OFS 0 +#define UDMA_DCQCN_AI_SZ sizeof(u16) +#define UDMA_DCQCN_AI_MAX ((u16) (~0U)) +#define UDMA_DCQCN_F_OFS (UDMA_DCQCN_AI_OFS + UDMA_DCQCN_AI_SZ) +#define UDMA_DCQCN_F_SZ sizeof(u8) +#define UDMA_DCQCN_F_MAX ((u8) (~0U)) +#define UDMA_DCQCN_TKP_OFS (UDMA_DCQCN_F_OFS + UDMA_DCQCN_F_SZ) +#define UDMA_DCQCN_TKP_SZ sizeof(u8) +#define UDMA_DCQCN_TKP_MAX 15 +#define UDMA_DCQCN_TMP_OFS (UDMA_DCQCN_TKP_OFS + UDMA_DCQCN_TKP_SZ) +#define UDMA_DCQCN_TMP_SZ sizeof(u16) +#define UDMA_DCQCN_TMP_MAX 15 +#define UDMA_DCQCN_ALP_OFS (UDMA_DCQCN_TMP_OFS + UDMA_DCQCN_TMP_SZ) +#define UDMA_DCQCN_ALP_SZ sizeof(u16) +#define UDMA_DCQCN_ALP_MAX ((u16) (~0U)) +#define UDMA_DCQCN_MAX_SPEED_OFS (UDMA_DCQCN_ALP_OFS + \ + UDMA_DCQCN_ALP_SZ) +#define UDMA_DCQCN_MAX_SPEED_SZ sizeof(u32) +#define UDMA_DCQCN_MAX_SPEED_MAX ((u32) (~0U)) +#define UDMA_DCQCN_G_OFS (UDMA_DCQCN_MAX_SPEED_OFS + \ + UDMA_DCQCN_MAX_SPEED_SZ) +#define UDMA_DCQCN_G_SZ sizeof(u8) +#define UDMA_DCQCN_G_MAX 15 +#define UDMA_DCQCN_AL_OFS (UDMA_DCQCN_G_OFS + UDMA_DCQCN_G_SZ) +#define UDMA_DCQCN_AL_SZ sizeof(u8) +#define UDMA_DCQCN_AL_MAX ((u8) (~0U)) +#define UDMA_DCQCN_CNP_TIME_OFS (UDMA_DCQCN_AL_OFS + \ + UDMA_DCQCN_AL_SZ) +#define UDMA_DCQCN_CNP_TIME_SZ sizeof(u8) +#define UDMA_DCQCN_CNP_TIME_MAX ((u8) (~0U)) +#define UDMA_DCQCN_ASHIFT_OFS (UDMA_DCQCN_CNP_TIME_OFS + \ + UDMA_DCQCN_CNP_TIME_SZ) +#define UDMA_DCQCN_ASHIFT_SZ sizeof(u8) +#define UDMA_DCQCN_ASHIFT_MAX 15 +#define UDMA_DCQCN_LIFESPAN_OFS (UDMA_DCQCN_ASHIFT_OFS + \ + UDMA_DCQCN_ASHIFT_SZ) +#define UDMA_DCQCN_LIFESPAN_SZ sizeof(u32) +#define UDMA_DCQCN_LIFESPAN_MAX 1000 + +#define UDMA_LDCP_CWD0_OFS 0 +#define UDMA_LDCP_CWD0_SZ sizeof(u32) +#define UDMA_LDCP_CWD0_MAX ((u32) (~0U)) +#define UDMA_LDCP_ALPHA_OFS (UDMA_LDCP_CWD0_OFS + UDMA_LDCP_CWD0_SZ) +#define UDMA_LDCP_ALPHA_SZ sizeof(u8) +#define UDMA_LDCP_ALPHA_MAX ((u8) (~0U)) +#define UDMA_LDCP_GAMMA_OFS (UDMA_LDCP_ALPHA_OFS + \ + UDMA_LDCP_ALPHA_SZ) +#define UDMA_LDCP_GAMMA_SZ sizeof(u8) +#define UDMA_LDCP_GAMMA_MAX ((u8) (~0U)) +#define UDMA_LDCP_BETA_OFS (UDMA_LDCP_GAMMA_OFS + \ + UDMA_LDCP_GAMMA_SZ) +#define UDMA_LDCP_BETA_SZ sizeof(u8) +#define UDMA_LDCP_BETA_MAX ((u8) (~0U)) +#define UDMA_LDCP_ETA_OFS (UDMA_LDCP_BETA_OFS + UDMA_LDCP_BETA_SZ) +#define UDMA_LDCP_ETA_SZ sizeof(u8) +#define UDMA_LDCP_ETA_MAX ((u8) (~0U)) +#define UDMA_LDCP_LIFESPAN_OFS (4 * sizeof(u32)) +#define UDMA_LDCP_LIFESPAN_SZ sizeof(u32) +#define UDMA_LDCP_LIFESPAN_MAX 1000 + +#define UDMA_HC3_INITIAL_WINDOW_OFS 0 +#define UDMA_HC3_INITIAL_WINDOW_SZ sizeof(u32) +#define UDMA_HC3_INITIAL_WINDOW_MAX ((u32) (~0U)) +#define UDMA_HC3_BANDWIDTH_OFS (UDMA_HC3_INITIAL_WINDOW_OFS + \ + UDMA_HC3_INITIAL_WINDOW_SZ) +#define UDMA_HC3_BANDWIDTH_SZ sizeof(u32) +#define UDMA_HC3_BANDWIDTH_MAX ((u32) (~0U)) +#define UDMA_HC3_QLEN_SHIFT_OFS (UDMA_HC3_BANDWIDTH_OFS + \ + UDMA_HC3_BANDWIDTH_SZ) +#define UDMA_HC3_QLEN_SHIFT_SZ sizeof(u8) +#define UDMA_HC3_QLEN_SHIFT_MAX ((u8) (~0U)) +#define UDMA_HC3_PORT_USAGE_SHIFT_OFS (UDMA_HC3_QLEN_SHIFT_OFS + \ + UDMA_HC3_QLEN_SHIFT_SZ) +#define UDMA_HC3_PORT_USAGE_SHIFT_SZ sizeof(u8) +#define UDMA_HC3_PORT_USAGE_SHIFT_MAX ((u8) (~0U)) +#define UDMA_HC3_OVER_PERIOD_OFS (UDMA_HC3_PORT_USAGE_SHIFT_OFS + \ + UDMA_HC3_PORT_USAGE_SHIFT_SZ) +#define UDMA_HC3_OVER_PERIOD_SZ sizeof(u8) +#define UDMA_HC3_OVER_PERIOD_MAX ((u8) (~0U)) +#define UDMA_HC3_MAX_STAGE_OFS (UDMA_HC3_OVER_PERIOD_OFS + \ + UDMA_HC3_OVER_PERIOD_SZ) +#define UDMA_HC3_MAX_STAGE_SZ sizeof(u8) +#define UDMA_HC3_MAX_STAGE_MAX ((u8) (~0U)) +#define UDMA_HC3_GAMMA_SHIFT_OFS (UDMA_HC3_MAX_STAGE_OFS + \ + UDMA_HC3_MAX_STAGE_SZ) +#define UDMA_HC3_GAMMA_SHIFT_SZ sizeof(u8) +#define UDMA_HC3_GAMMA_SHIFT_MAX 15 +#define UDMA_HC3_LIFESPAN_OFS (4 * sizeof(u32)) +#define UDMA_HC3_LIFESPAN_SZ sizeof(u32) +#define UDMA_HC3_LIFESPAN_MAX 1000 + +#define UDMA_DIP_AI_OFS 0 +#define UDMA_DIP_AI_SZ sizeof(u16) +#define UDMA_DIP_AI_MAX ((u16) (~0U)) +#define UDMA_DIP_F_OFS (UDMA_DIP_AI_OFS + UDMA_DIP_AI_SZ) +#define UDMA_DIP_F_SZ sizeof(u8) +#define UDMA_DIP_F_MAX ((u8) (~0U)) +#define UDMA_DIP_TKP_OFS (UDMA_DIP_F_OFS + UDMA_DIP_F_SZ) +#define UDMA_DIP_TKP_SZ sizeof(u8) +#define UDMA_DIP_TKP_MAX 15 +#define UDMA_DIP_TMP_OFS (UDMA_DIP_TKP_OFS + UDMA_DIP_TKP_SZ) +#define UDMA_DIP_TMP_SZ sizeof(u16) +#define UDMA_DIP_TMP_MAX 15 +#define UDMA_DIP_ALP_OFS (UDMA_DIP_TMP_OFS + UDMA_DIP_TMP_SZ) +#define UDMA_DIP_ALP_SZ sizeof(u16) +#define UDMA_DIP_ALP_MAX ((u16) (~0U)) +#define UDMA_DIP_MAX_SPEED_OFS (UDMA_DIP_ALP_OFS + UDMA_DIP_ALP_SZ) +#define UDMA_DIP_MAX_SPEED_SZ sizeof(u32) +#define UDMA_DIP_MAX_SPEED_MAX ((u32) (~0U)) +#define UDMA_DIP_G_OFS (UDMA_DIP_MAX_SPEED_OFS + \ + UDMA_DIP_MAX_SPEED_SZ) +#define UDMA_DIP_G_SZ sizeof(u8) +#define UDMA_DIP_G_MAX 15 +#define UDMA_DIP_AL_OFS (UDMA_DIP_G_OFS + UDMA_DIP_G_SZ) +#define UDMA_DIP_AL_SZ sizeof(u8) +#define UDMA_DIP_AL_MAX ((u8) (~0U)) +#define UDMA_DIP_CNP_TIME_OFS (UDMA_DIP_AL_OFS + UDMA_DIP_AL_SZ) +#define UDMA_DIP_CNP_TIME_SZ sizeof(u8) +#define UDMA_DIP_CNP_TIME_MAX ((u8) (~0U)) +#define UDMA_DIP_ASHIFT_OFS (UDMA_DIP_CNP_TIME_OFS + \ + UDMA_DIP_CNP_TIME_SZ) +#define UDMA_DIP_ASHIFT_SZ sizeof(u8) +#define UDMA_DIP_ASHIFT_MAX 15 +#define UDMA_DIP_LIFESPAN_OFS (UDMA_DIP_ASHIFT_OFS + \ + UDMA_DIP_ASHIFT_SZ) +#define UDMA_DIP_LIFESPAN_SZ sizeof(u32) +#define UDMA_DIP_LIFESPAN_MAX 1000 + +#define ATTR_RW_RONLY_RONLY 0644 + +struct udma_port_attribute { + struct attribute attr; + ssize_t (*show)(struct udma_port *pdata, + struct udma_port_attribute *attr, char *buf); + ssize_t (*store)(struct udma_port *pdata, + struct udma_port_attribute *attr, const char *buf, + size_t count); +}; + +struct udma_port_cc_attr { + struct udma_port_attribute port_attr; + enum udma_cong_type algo_type; + uint32_t offset; + uint32_t size; + uint32_t min; + uint32_t max; +}; + +int udma_register_cc_sysfs(struct udma_dev *udma_dev); +void udma_unregister_cc_sysfs(struct udma_dev *udma_dev); + +#endif /* _UDMA_SYSFS_H */ diff --git a/drivers/ub/hw/hns3/hns3_udma_tp.c b/drivers/ub/hw/hns3/hns3_udma_tp.c new file mode 100644 index 0000000000000000000000000000000000000000..ef2eb335fb87a5f163cadc0ecb5ab61b7304a1bd --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_tp.c @@ -0,0 +1,494 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#include +#include +#include "hns3_udma_jfr.h" +#include "hns3_udma_jfs.h" +#include "hns3_udma_jetty.h" +#include "hns3_udma_hem.h" +#include "hns3_udma_dca.h" +#include "hns3_udma_dfx.h" +#include "hns3_udma_tp.h" +struct udma_qp *get_qp(struct udma_dev *udma_device, uint32_t qpn) +{ + return (struct udma_qp *)xa_load(&udma_device->qp_table.xa, qpn); +} + +static enum udma_qp_state to_udma_qp_state(enum ubcore_tp_state state) +{ + switch (state) { + case UBCORE_TP_STATE_RESET: + return QPS_RESET; + case UBCORE_TP_STATE_RTR: + return QPS_RTR; + case UBCORE_TP_STATE_RTS: + return QPS_RTS; + case UBCORE_TP_STATE_ERROR: + return QPS_ERR; + default: + return QPS_ERR; + } +} + +struct udma_modify_tp_attr *udma_get_m_attr(struct ubcore_tp *tp, struct udma_qp *qp, + const struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask mask) +{ + struct udma_modify_tp_attr *m_attr; + + m_attr = kzalloc(sizeof(*m_attr), GFP_KERNEL); + if (m_attr == NULL) + return NULL; + + memcpy(m_attr->dmac, tp->peer_net_addr.mac, sizeof(m_attr->dmac)); + m_attr->data_udp_start = tp->data_udp_start; + m_attr->ack_udp_start = tp->ack_udp_start; + m_attr->udp_range = tp->udp_range; + m_attr->hop_limit = MAX_HOP_LIMIT; + m_attr->sgid_index = 0; + *(uint32_t *)(&m_attr->dipv4) = *(uint32_t *)(tp->peer_eid.raw + + SGID_H_SHIFT); + memcpy(m_attr->dgid, tp->peer_eid.raw, sizeof(tp->peer_eid.raw)); + + if (!qp->qp_attr.is_tgt) { + m_attr->retry_cnt = qp->retry_cnt; + m_attr->ack_timeout = qp->ack_timeout; + m_attr->rnr_retry = qp->rnr_retry; + m_attr->priority = qp->priority; + if (qp->qp_attr.is_jetty) + m_attr->min_rnr_timer = qp->min_rnr_timer; + } else { + m_attr->min_rnr_timer = qp->min_rnr_timer; + if (qp->qp_attr.is_jetty) { + m_attr->retry_cnt = qp->retry_cnt; + m_attr->ack_timeout = qp->ack_timeout; + m_attr->rnr_retry = qp->rnr_retry; + m_attr->priority = qp->priority; + } + } + + return m_attr; +} + +int udma_modify_tp(struct ubcore_tp *tp, const struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask mask) +{ + struct udma_modify_tp_attr *m_attr; + enum udma_qp_state target_state; + enum udma_qp_state curr_state; + struct udma_dev *udma_device; + struct udma_tp *utp; + struct udma_qp *qp; + int ret = -EINVAL; + + udma_device = to_udma_dev(tp->ub_dev); + utp = to_udma_tp(tp); + + qp = &utp->qp; + if (!qp) + goto error; + + curr_state = to_udma_qp_state(tp->state); + + if (mask.bs.state) + target_state = to_udma_qp_state(attr->state); + else + target_state = QPS_ERR; + + m_attr = udma_get_m_attr(tp, qp, attr, mask); + if (!m_attr) + goto error; + + qp->udma_device = udma_device; + qp->send_jfc = qp->qp_attr.send_jfc; + qp->recv_jfc = qp->qp_attr.recv_jfc; + qp->m_attr = m_attr; + if (attr) + qp->ubcore_path_mtu = attr->mtu; + ret = udma_modify_qp_common(qp, attr, mask, curr_state, target_state); + kfree(m_attr); +error: + return ret; +} + +void *udma_erase_tp(struct udma_tp *udma_tp) +{ + struct udma_qp_attr *qp_attr; + struct udma_jetty *jetty; + struct udma_tp *pre_tp; + struct udma_jfr *jfr; + struct udma_jfs *jfs; + uint32_t hash; + + qp_attr = &udma_tp->qp.qp_attr; + jetty = qp_attr->jetty; + jfr = qp_attr->jfr; + jfs = qp_attr->jfs; + + if (qp_attr->is_jetty) { + if (jetty->tp_mode == UBCORE_TP_RM) { + hash = udma_get_jetty_hash(&udma_tp->tjetty_id); + pre_tp = (struct udma_tp *)xa_load(&jetty->srm_node_table, + hash); + if (!pre_tp) + return NULL; + + if (pre_tp->qp.qpn == udma_tp->qp.qpn) + xa_erase(&jetty->srm_node_table, hash); + } else if (jetty->tp_mode == UBCORE_TP_RC) { + if (jetty->rc_node.tp == NULL) + return NULL; + + if (jetty->rc_node.tpn == udma_tp->ubcore_tp.tpn) { + jetty->rc_node.tp = NULL; + jetty->rc_node.tpn = 0; + } + } + } else { + if (jfr) + return xa_erase(&jfr->tp_table_xa, + udma_tp->ubcore_tp.tpn); + else if (jfs) + return xa_erase(&jfs->node_table, + udma_tp->ubcore_tp.tpn); + } + + return udma_tp; +} + +static void store_tpn(struct udma_dev *udma_device, struct udma_tp *tp) +{ + struct tpn_list *tpn_new; + struct tpn_list *tpn_now; + unsigned long flags; + spinlock_t *lock; + int ret; + int i; + + ret = udma_find_dfx_dev(udma_device, &i); + if (ret) + return; + + tpn_new = kzalloc(sizeof(struct tpn_list), GFP_KERNEL); + if (tpn_new == NULL) + return; + + lock = &g_udma_dfx_list[i].dfx->tpn_list->node_lock; + spin_lock_irqsave(lock, flags); + list_for_each_entry(tpn_now, + &g_udma_dfx_list[i].dfx->tpn_list->node, node) { + if (tpn_now->tpn == tp->ubcore_tp.tpn) + goto found; + } + + tpn_new->tpn = tp->ubcore_tp.tpn; + list_add(&tpn_new->node, &g_udma_dfx_list[i].dfx->tpn_list->node); + spin_unlock_irqrestore(lock, flags); + + return; + +found: + spin_unlock_irqrestore(lock, flags); + kfree(tpn_new); +} + +static void delete_tpn(struct udma_dev *udma_device, struct ubcore_tp *tp) +{ + struct tpn_list *tpn_now, *tpn_tmp; + unsigned long flags; + spinlock_t *lock; + int ret; + int i; + + ret = udma_find_dfx_dev(udma_device, &i); + if (ret) + return; + + lock = &g_udma_dfx_list[i].dfx->tpn_list->node_lock; + spin_lock_irqsave(lock, flags); + list_for_each_entry_safe(tpn_now, tpn_tmp, + &g_udma_dfx_list[i].dfx->tpn_list->node, + node) { + if (tpn_now->tpn == tp->tpn) { + list_del(&tpn_now->node); + kfree(tpn_now); + spin_unlock_irqrestore(lock, flags); + return; + } + } + spin_unlock_irqrestore(lock, flags); +} + + +int udma_destroy_tp(struct ubcore_tp *tp) +{ + struct udma_dev *udma_device = to_udma_dev(tp->ub_dev); + union ubcore_tp_attr_mask ubcore_attr_mask; + enum udma_qp_state curr_state; + struct udma_tp *udma_tp; + struct udma_qp *qp; + int ret = 0; + + udma_tp = to_udma_tp(tp); + if (!udma_erase_tp(udma_tp)) { + dev_err(udma_device->dev, + "failed to find tp, tpn = 0x%x\n", tp->tpn); + return 0; + } + + if (dfx_switch) + delete_tpn(udma_device, tp); + + qp = &udma_tp->qp; + curr_state = to_udma_qp_state(tp->state); + ubcore_attr_mask.value = 0; + qp->m_attr = NULL; + + if (qp->state != QPS_RESET) { + ret = udma_modify_qp_common(qp, NULL, ubcore_attr_mask, curr_state, QPS_RESET); + if (ret) { + dev_err(udma_device->dev, + "Modify QP 0x%06llx to Reset failed(%d).\n", + qp->qpn, ret); + goto error; + } + } + + udma_destroy_qp_common(udma_device, qp); + + kfree(udma_tp); + + return ret; + +error: + return -EINVAL; +} + +static void udma_set_tp(struct ubcore_device *dev, const struct ubcore_tp_cfg *cfg, + struct udma_tp *tp) +{ + tp->ubcore_tp.tpn = tp->qp.qpn; + tp->ubcore_tp.ub_dev = dev; + tp->ubcore_tp.flag.bs.target = cfg->flag.bs.target; + tp->ubcore_tp.flag.bs.oor_en = cfg->flag.bs.oor_en; + tp->ubcore_tp.flag.bs.sr_en = cfg->flag.bs.sr_en; + tp->ubcore_tp.flag.bs.spray_en = cfg->flag.bs.spray_en; + tp->ubcore_tp.local_net_addr = cfg->local_net_addr; + tp->ubcore_tp.peer_net_addr = cfg->peer_net_addr; + tp->ubcore_tp.local_eid = cfg->local_eid; + tp->ubcore_tp.peer_eid = cfg->peer_eid; + tp->ubcore_tp.trans_mode = cfg->trans_mode; + tp->ubcore_tp.rx_psn = cfg->rx_psn; + tp->ubcore_tp.mtu = cfg->mtu; + tp->ubcore_tp.data_udp_start = cfg->data_udp_start; + tp->ubcore_tp.ack_udp_start = cfg->ack_udp_start; + tp->ubcore_tp.udp_range = cfg->udp_range; + tp->ubcore_tp.retry_num = cfg->retry_num; + tp->ubcore_tp.ack_timeout = cfg->ack_timeout; + tp->ubcore_tp.tc = cfg->tc; + tp->ubcore_tp.state = UBCORE_TP_STATE_RESET; +} + +static void copy_attr_to_pre_tp(struct udma_dev *udma_device, bool is_rm, + struct udma_qp *from_qp, struct udma_qp *to_qp) +{ + if (from_qp->qp_attr.is_tgt) + return; + + to_qp->sge = from_qp->sge; + to_qp->sq = from_qp->sq; + to_qp->sdb = from_qp->sdb; + to_qp->priority = from_qp->priority; + to_qp->dca_ctx = from_qp->dca_ctx; + to_qp->en_flags = from_qp->en_flags; + to_qp->buff_size = from_qp->buff_size; + if (to_qp->en_flags & UDMA_QP_CAP_DYNAMIC_CTX_ATTACH) + udma_enable_dca(udma_device, to_qp); + + udma_mtr_move(&from_qp->mtr, &to_qp->mtr); + if (is_rm) + copy_send_jfc(from_qp, to_qp); + + to_qp->qp_attr.cap.max_send_wr = from_qp->qp_attr.cap.max_send_wr; + to_qp->qp_attr.cap.max_send_sge = from_qp->qp_attr.cap.max_send_sge; + to_qp->qp_attr.cap.max_inline_data = + from_qp->qp_attr.cap.max_inline_data; + + from_qp->no_free_wqe_buf = true; + to_qp->force_free_wqe_buf = true; +} + +static int udma_store_jetty_tp(struct udma_dev *udma_device, + struct udma_jetty *jetty, struct udma_tp *tp, + struct ubcore_tp **fail_ret_tp) +{ + struct udma_tp *pre_tp; + uint32_t tjetty_hash; + uint32_t hash; + int ret = 0; + + hash = udma_get_jetty_hash(&tp->tjetty_id); + if (jetty->tp_mode == UBCORE_TP_RM) { + pre_tp = (struct udma_tp *)xa_load(&jetty->srm_node_table, hash); + if (pre_tp) { + copy_attr_to_pre_tp(udma_device, true, &tp->qp, + &pre_tp->qp); + *fail_ret_tp = &pre_tp->ubcore_tp; + return 0; + } + + ret = xa_err(xa_store(&jetty->srm_node_table, + hash, tp, GFP_KERNEL)); + if (ret) + dev_err(udma_device->dev, + "failed store jetty tp xarray, ret = %d\n", ret); + } else if (jetty->tp_mode == UBCORE_TP_RC) { + if (jetty->rc_node.tp == NULL) { + jetty->rc_node.tp = tp; + jetty->rc_node.tpn = tp->ubcore_tp.tpn; + jetty->rc_node.tjetty_id = tp->tjetty_id; + } else { + tjetty_hash = + udma_get_jetty_hash(&jetty->rc_node.tjetty_id); + if (tjetty_hash == hash && + (tp->qp.en_flags & UDMA_QP_CAP_DYNAMIC_CTX_ATTACH)) { + copy_attr_to_pre_tp(udma_device, false, &tp->qp, + &jetty->rc_node.tp->qp); + *fail_ret_tp = &jetty->rc_node.tp->ubcore_tp; + } else if (tjetty_hash == hash) { + *fail_ret_tp = &jetty->rc_node.tp->ubcore_tp; + } else { + dev_err(udma_device->dev, + "jetty has bind a target jetty, jetty_id = %d.\n", + jetty->rc_node.tjetty_id.id); + return -EEXIST; + } + } + } + + return ret; +} + +static int udma_store_tp(struct udma_dev *udma_device, struct udma_tp *tp, + struct ubcore_tp **fail_ret_tp) +{ + struct udma_qp_attr *qp_attr; + struct udma_jetty *jetty; + struct udma_jfr *jfr; + struct udma_jfs *jfs; + int ret = 0; + + qp_attr = &tp->qp.qp_attr; + jfr = qp_attr->jfr; + jfs = qp_attr->jfs; + jetty = qp_attr->jetty; + + if (qp_attr->is_jetty) { + ret = udma_store_jetty_tp(udma_device, jetty, tp, fail_ret_tp); + if (ret) { + dev_err(udma_device->dev, + "failed store jetty tp, ret = %d\n", ret); + return ret; + } + } else { + if (jfr) { + ret = xa_err(xa_store(&jfr->tp_table_xa, + tp->ubcore_tp.tpn, tp, + GFP_KERNEL)); + if (ret) { + dev_err(udma_device->dev, + "failed store jfr tp, ret = %d\n", ret); + return ret; + } + } else if (jfs) { + ret = xa_err(xa_store(&jfs->node_table, + tp->ubcore_tp.tpn, tp, + GFP_KERNEL)); + if (ret) { + dev_err(udma_device->dev, + "failed store jfs tp, ret = %d\n", ret); + return ret; + } + } + } + + return ret; +} + +static void lock_jetty(struct udma_qp_attr *qp_attr) +{ + struct udma_jetty *jetty = qp_attr->jetty; + + if (qp_attr->is_jetty) + mutex_lock(&jetty->tp_mutex); +} + +static void unlock_jetty(struct udma_qp_attr *qp_attr) +{ + struct udma_jetty *jetty = qp_attr->jetty; + + if (qp_attr->is_jetty) + mutex_unlock(&jetty->tp_mutex); +} + +struct ubcore_tp *udma_create_tp(struct ubcore_device *dev, const struct ubcore_tp_cfg *cfg, + struct ubcore_udata *udata) +{ + struct udma_dev *udma_dev = to_udma_dev(dev); + struct ubcore_tp *fail_ret_tp = NULL; + struct udma_tp *tp; + int ret; + + tp = kzalloc(sizeof(*tp), GFP_KERNEL); + if (!tp) + return ERR_PTR(-ENOMEM); + + ret = udma_fill_qp_attr(udma_dev, &tp->qp.qp_attr, cfg, udata); + if (ret) { + dev_err(udma_dev->dev, "failed to fill qp attr.\n"); + goto failed_alloc_tp; + } + tp->tjetty_id.id = tp->qp.qp_attr.tgt_id; + tp->tjetty_id.eid = cfg->peer_eid; + + lock_jetty(&tp->qp.qp_attr); + ret = udma_create_qp_common(udma_dev, &tp->qp, udata); + if (ret) { + dev_err(udma_dev->dev, + "Failed to create qp common with ret is %d.\n", ret); + unlock_jetty(&tp->qp.qp_attr); + goto failed_alloc_tp; + } + + udma_set_tp(dev, cfg, tp); + + ret = udma_store_tp(udma_dev, tp, &fail_ret_tp); + unlock_jetty(&tp->qp.qp_attr); + if (ret || fail_ret_tp) + goto failed_create_qp; + + if (dfx_switch) + store_tpn(udma_dev, tp); + + return &tp->ubcore_tp; + +failed_create_qp: + udma_destroy_qp_common(udma_dev, &tp->qp); +failed_alloc_tp: + kfree(tp); + + return fail_ret_tp; +} diff --git a/drivers/ub/hw/hns3/hns3_udma_tp.h b/drivers/ub/hw/hns3/hns3_udma_tp.h new file mode 100644 index 0000000000000000000000000000000000000000..84781db93ad7571f7146c50cc54991dcb380ccbe --- /dev/null +++ b/drivers/ub/hw/hns3/hns3_udma_tp.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Huawei UDMA Linux driver + * Copyright (c) 2023-2023 Hisilicon Limited. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + */ + +#ifndef _UDMA_TP_H +#define _UDMA_TP_H + +#include +#include "hns3_udma_qp.h" + +#define MAX_HOP_LIMIT 255 + +struct udma_tp { + struct ubcore_tp ubcore_tp; + struct udma_qp qp; + struct ubcore_jetty_id tjetty_id; +}; + +static inline struct udma_tp *to_udma_tp(struct ubcore_tp *ubcore_tp) +{ + return container_of(ubcore_tp, struct udma_tp, ubcore_tp); +} + +static inline uint32_t udma_get_jetty_hash(const struct ubcore_jetty_id *jetty_id) +{ + return jhash(jetty_id, sizeof(struct ubcore_jetty_id), 0); +} + +struct ubcore_tp *udma_create_tp(struct ubcore_device *dev, + const struct ubcore_tp_cfg *cfg, + struct ubcore_udata *udata); +int udma_destroy_tp(struct ubcore_tp *tp); +int udma_modify_tp(struct ubcore_tp *tp, const struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask mask); +struct udma_qp *get_qp(struct udma_dev *udma_device, uint32_t qpn); + +#endif /* _UDMA_TP_H */