diff --git a/Documentation/ABI/testing/debugfs-hisi-hpre b/Documentation/ABI/testing/debugfs-hisi-hpre index 77275ac6cdb36ab6666426b3ef5ce0336cbaa8fb..2923dd9cf997517e2b16ff5c67b5b7161e593146 100644 --- a/Documentation/ABI/testing/debugfs-hisi-hpre +++ b/Documentation/ABI/testing/debugfs-hisi-hpre @@ -109,6 +109,13 @@ Description: Dump the stop queue status of the QM. The default value is 0, will return non-zero value. Available for both PF and VF, and take no other effect on HPRE. +What: /sys/kernel/debug/hisi_hpre//qm/qm_state +Date: Dec 2023 +Contact: qianweili@huawei.com +Description: Dump the status of the device. + 0: busy, 1: idle. + Only available for PF, and take no other effect on HPRE. + What: /sys/kernel/debug/hisi_hpre//hpre_dfx/send_cnt Date: Apr 2020 Contact: linux-crypto@vger.kernel.org diff --git a/Documentation/ABI/testing/debugfs-hisi-sec b/Documentation/ABI/testing/debugfs-hisi-sec index 91c144331ac066bff55ab8198c9a3e501c1b4112..623ddf7ece81c5b78cc8e32b028d4368750aa7e8 100644 --- a/Documentation/ABI/testing/debugfs-hisi-sec +++ b/Documentation/ABI/testing/debugfs-hisi-sec @@ -89,6 +89,13 @@ Description: Dump the stop queue status of the QM. The default value is 0, will return non-zero value. Available for both PF and VF, and take no other effect on SEC. +What: /sys/kernel/debug/hisi_sec2//qm/qm_state +Date: Dec 2023 +Contact: qianweili@huawei.com +Description: Dump the status of the device. + 0: busy, 1: idle. + Only available for PF, and take no other effect on SEC. + What: /sys/kernel/debug/hisi_sec2//sec_dfx/send_cnt Date: Apr 2020 Contact: linux-crypto@vger.kernel.org diff --git a/Documentation/ABI/testing/debugfs-hisi-zip b/Documentation/ABI/testing/debugfs-hisi-zip index 9b602880259dc8f01311a138e29db3b0c562b7a1..ae3d2d85f7501bc51d639c6d16379437c476942d 100644 --- a/Documentation/ABI/testing/debugfs-hisi-zip +++ b/Documentation/ABI/testing/debugfs-hisi-zip @@ -102,6 +102,13 @@ Description: Dump the stop queue status of the QM. The default value is 0, will return non-zero value. Available for both PF and VF, and take no other effect on ZIP. +What: /sys/kernel/debug/hisi_zip//qm/qm_state +Date: Dec 2023 +Contact: qianweili@huawei.com +Description: Dump the status of the device. + 0: busy, 1: idle. + Only available for PF, and take no other effect on ZIP. + What: /sys/kernel/debug/hisi_zip//zip_dfx/send_cnt Date: Apr 2020 Contact: linux-crypto@vger.kernel.org diff --git a/drivers/crypto/hisilicon/debugfs.c b/drivers/crypto/hisilicon/debugfs.c index b34806a92410d264173e28feea44d82fec7ca50c..c423bf19d985e7ae72b74f50d93b7737783aad43 100644 --- a/drivers/crypto/hisilicon/debugfs.c +++ b/drivers/crypto/hisilicon/debugfs.c @@ -24,6 +24,8 @@ #define QM_DFX_QN_SHIFT 16 #define QM_DFX_CNT_CLR_CE 0x100118 #define QM_DBG_WRITE_LEN 1024 +#define QM_IN_IDLE_ST_REG 0x1040e4 +#define QM_IN_IDLE_STATE 0x1 static const char * const qm_debug_file_name[] = { [CURRENT_QM] = "current_qm", @@ -81,6 +83,30 @@ static const struct debugfs_reg32 qm_dfx_regs[] = { {"QM_DFX_FF_ST5 ", 0x1040dc}, {"QM_DFX_FF_ST6 ", 0x1040e0}, {"QM_IN_IDLE_ST ", 0x1040e4}, + {"QM_CACHE_CTL ", 0x100050}, + {"QM_TIMEOUT_CFG ", 0x100070}, + {"QM_DB_TIMEOUT_CFG ", 0x100074}, + {"QM_FLR_PENDING_TIME_CFG ", 0x100078}, + {"QM_ARUSR_MCFG1 ", 0x100088}, + {"QM_AWUSR_MCFG1 ", 0x100098}, + {"QM_AXI_M_CFG_ENABLE ", 0x1000B0}, + {"QM_RAS_CE_THRESHOLD ", 0x1000F8}, + {"QM_AXI_TIMEOUT_CTRL ", 0x100120}, + {"QM_AXI_TIMEOUT_STATUS ", 0x100124}, + {"QM_CQE_AGGR_TIMEOUT_CTRL ", 0x100144}, + {"ACC_RAS_MSI_INT_SEL ", 0x1040fc}, + {"QM_CQE_OUT ", 0x104100}, + {"QM_EQE_OUT ", 0x104104}, + {"QM_AEQE_OUT ", 0x104108}, + {"QM_DB_INFO0 ", 0x104180}, + {"QM_DB_INFO1 ", 0x104184}, + {"QM_AM_CTRL_GLOBAL ", 0x300000}, + {"QM_AM_CURR_PORT_STS ", 0x300100}, + {"QM_AM_CURR_TRANS_RETURN ", 0x300150}, + {"QM_AM_CURR_RD_MAX_TXID ", 0x300154}, + {"QM_AM_CURR_WR_MAX_TXID ", 0x300158}, + {"QM_AM_ALARM_RRESP ", 0x300180}, + {"QM_AM_ALARM_BRESP ", 0x300184}, }; static const struct debugfs_reg32 qm_vf_dfx_regs[] = { @@ -1001,6 +1027,30 @@ static int qm_diff_regs_show(struct seq_file *s, void *unused) } DEFINE_SHOW_ATTRIBUTE(qm_diff_regs); +static int qm_state_show(struct seq_file *s, void *unused) +{ + struct hisi_qm *qm = s->private; + u32 val; + int ret; + + ret = hisi_qm_get_dfx_access(qm); + if (!ret) { + val = readl(qm->io_base + QM_IN_IDLE_ST_REG); + hisi_qm_put_dfx_access(qm); + } else if (ret == -EAGAIN) { + /* If device is in suspended, directly return the idle state. */ + val = QM_IN_IDLE_STATE; + } else { + return ret; + } + + seq_printf(s, "%u\n", val); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(qm_state); + static ssize_t qm_status_read(struct file *filp, char __user *buffer, size_t count, loff_t *pos) { @@ -1073,6 +1123,9 @@ void hisi_qm_debug_init(struct hisi_qm *qm) /* only show this in PF */ if (qm->fun_type == QM_HW_PF) { + debugfs_create_file("qm_state", 0444, qm->debug.qm_d, + qm, &qm_state_fops); + qm_create_debugfs_file(qm, qm->debug.debug_root, CURRENT_QM); for (i = CURRENT_Q; i < DEBUG_FILE_NUM; i++) qm_create_debugfs_file(qm, qm->debug.qm_d, i); diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index 44d3eedfb7556b01f9aa698b8fec295e03fdf834..7829db84d29b32e536c95de2a8fcda113dc77eaf 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -13,9 +13,7 @@ #include #include "hpre.h" -#define HPRE_QM_ABNML_INT_MASK 0x100004 #define HPRE_CTRL_CNT_CLR_CE_BIT BIT(0) -#define HPRE_COMM_CNT_CLR_CE 0x0 #define HPRE_CTRL_CNT_CLR_CE 0x301000 #define HPRE_FSM_MAX_CNT 0x301008 #define HPRE_VFG_AXQOS 0x30100c @@ -42,7 +40,6 @@ #define HPRE_HAC_INT_SET 0x301500 #define HPRE_RNG_TIMEOUT_NUM 0x301A34 #define HPRE_CORE_INT_ENABLE 0 -#define HPRE_CORE_INT_DISABLE GENMASK(21, 0) #define HPRE_RDCHN_INI_ST 0x301a00 #define HPRE_CLSTR_BASE 0x302000 #define HPRE_CORE_EN_OFFSET 0x04 @@ -66,7 +63,6 @@ #define HPRE_CLSTR_ADDR_INTRVL 0x1000 #define HPRE_CLUSTER_INQURY 0x100 #define HPRE_CLSTR_ADDR_INQRY_RSLT 0x104 -#define HPRE_TIMEOUT_ABNML_BIT 6 #define HPRE_PASID_EN_BIT 9 #define HPRE_REG_RD_INTVRL_US 10 #define HPRE_REG_RD_TMOUT_US 1000 @@ -203,9 +199,9 @@ static const struct hisi_qm_cap_info hpre_basic_info[] = { {HPRE_QM_RESET_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0xC37, 0x6C37}, {HPRE_QM_OOO_SHUTDOWN_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0x4, 0x6C37}, {HPRE_QM_CE_MASK_CAP, 0x312C, 0, GENMASK(31, 0), 0x0, 0x8, 0x8}, - {HPRE_NFE_MASK_CAP, 0x3130, 0, GENMASK(31, 0), 0x0, 0x3FFFFE, 0x1FFFFFE}, - {HPRE_RESET_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x3FFFFE, 0xBFFFFE}, - {HPRE_OOO_SHUTDOWN_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x22, 0xBFFFFE}, + {HPRE_NFE_MASK_CAP, 0x3130, 0, GENMASK(31, 0), 0x0, 0x3FFFFE, 0x1FFFC3E}, + {HPRE_RESET_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x3FFFFE, 0xBFFC3E}, + {HPRE_OOO_SHUTDOWN_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x22, 0xBFFC3E}, {HPRE_CE_MASK_CAP, 0x3138, 0, GENMASK(31, 0), 0x0, 0x1, 0x1}, {HPRE_CLUSTER_NUM_CAP, 0x313c, 20, GENMASK(3, 0), 0x0, 0x4, 0x1}, {HPRE_CORE_TYPE_NUM_CAP, 0x313c, 16, GENMASK(3, 0), 0x0, 0x2, 0x2}, @@ -358,6 +354,8 @@ static struct dfx_diff_registers hpre_diff_regs[] = { }, }; +static const struct hisi_qm_err_ini hpre_err_ini; + bool hpre_check_alg_support(struct hisi_qm *qm, u32 alg) { u32 cap_val; @@ -654,11 +652,6 @@ static int hpre_set_user_domain_and_cache(struct hisi_qm *qm) writel(HPRE_QM_USR_CFG_MASK, qm->io_base + QM_AWUSER_M_CFG_ENABLE); writel_relaxed(HPRE_QM_AXI_CFG_MASK, qm->io_base + QM_AXI_M_CFG); - /* HPRE need more time, we close this interrupt */ - val = readl_relaxed(qm->io_base + HPRE_QM_ABNML_INT_MASK); - val |= BIT(HPRE_TIMEOUT_ABNML_BIT); - writel_relaxed(val, qm->io_base + HPRE_QM_ABNML_INT_MASK); - if (qm->ver >= QM_HW_V3) writel(HPRE_RSA_ENB | HPRE_ECC_ENB, qm->io_base + HPRE_TYPES_ENB); @@ -667,9 +660,7 @@ static int hpre_set_user_domain_and_cache(struct hisi_qm *qm) writel(HPRE_QM_VFG_AX_MASK, qm->io_base + HPRE_VFG_AXCACHE); writel(0x0, qm->io_base + HPRE_BD_ENDIAN); - writel(0x0, qm->io_base + HPRE_INT_MASK); writel(0x0, qm->io_base + HPRE_POISON_BYPASS); - writel(0x0, qm->io_base + HPRE_COMM_CNT_CLR_CE); writel(0x0, qm->io_base + HPRE_ECC_BYPASS); writel(HPRE_BD_USR_MASK, qm->io_base + HPRE_BD_ARUSR_CFG); @@ -759,7 +750,7 @@ static void hpre_hw_error_disable(struct hisi_qm *qm) static void hpre_hw_error_enable(struct hisi_qm *qm) { - u32 ce, nfe; + u32 ce, nfe, err_en; ce = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_CE_MASK_CAP, qm->cap_ver); nfe = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_NFE_MASK_CAP, qm->cap_ver); @@ -776,7 +767,8 @@ static void hpre_hw_error_enable(struct hisi_qm *qm) hpre_master_ooo_ctrl(qm, true); /* enable hpre hw error interrupts */ - writel(HPRE_CORE_INT_ENABLE, qm->io_base + HPRE_INT_MASK); + err_en = ce | nfe | HPRE_HAC_RAS_FE_ENABLE; + writel(~err_en, qm->io_base + HPRE_INT_MASK); } static inline struct hisi_qm *hpre_file_to_qm(struct hpre_debugfs_file *file) @@ -1162,6 +1154,7 @@ static int hpre_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) qm->qp_num = pf_q_num; qm->debug.curr_qm_qp_num = pf_q_num; qm->qm_list = &hpre_devices; + qm->err_ini = &hpre_err_ini; if (pf_q_num_flag) set_bit(QM_MODULE_PARAM, &qm->misc_ctl); } @@ -1357,6 +1350,17 @@ static const struct hisi_qm_err_ini hpre_err_ini = { .err_info_init = hpre_err_info_init, }; +static void hpre_probe_uninit(struct hisi_qm *qm) +{ + if (qm->fun_type == QM_HW_VF) + return; + + hpre_cnt_regs_clear(qm); + qm->debug.curr_qm_qp_num = 0; + hpre_show_last_regs_uninit(qm); + hpre_close_sva_prefetch(qm); +} + static int hpre_pf_probe_init(struct hpre *hpre) { struct hisi_qm *qm = &hpre->qm; @@ -1367,10 +1371,6 @@ static int hpre_pf_probe_init(struct hpre *hpre) return ret; hpre_open_sva_prefetch(qm); - - qm->err_ini = &hpre_err_ini; - qm->err_ini->err_info_init(qm); - hisi_qm_dev_err_init(qm); ret = hpre_show_last_regs_init(qm); if (ret) pci_err(qm->pdev, "Failed to init last word regs!\n"); @@ -1409,6 +1409,7 @@ static int hpre_probe(struct pci_dev *pdev, const struct pci_device_id *id) return -ENOMEM; qm = &hpre->qm; + set_bit(QM_DRIVER_DOWN, &qm->misc_ctl); ret = hpre_qm_init(qm, pdev); if (ret) { pci_err(pdev, "Failed to init HPRE QM (%d)!\n", ret); @@ -1425,6 +1426,9 @@ static int hpre_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) goto err_with_err_init; + /* Device is enabled, clear the flag. */ + clear_bit(QM_DRIVER_DOWN, &qm->misc_ctl); + ret = hpre_debugfs_init(qm); if (ret) dev_warn(&pdev->dev, "init debugfs fail!\n"); @@ -1457,13 +1461,12 @@ static int hpre_probe(struct pci_dev *pdev, const struct pci_device_id *id) err_qm_del_list: hisi_qm_del_list(qm, &hpre_devices); + hisi_qm_wait_task_finish(qm, &hpre_devices); hpre_debugfs_exit(qm); hisi_qm_stop(qm, QM_NORMAL); err_with_err_init: - hpre_show_last_regs_uninit(qm); - hisi_qm_dev_err_uninit(qm); - + hpre_probe_uninit(qm); err_with_qm_init: hisi_qm_uninit(qm); @@ -1483,14 +1486,7 @@ static void hpre_remove(struct pci_dev *pdev) hpre_debugfs_exit(qm); hisi_qm_stop(qm, QM_NORMAL); - - if (qm->fun_type == QM_HW_PF) { - hpre_cnt_regs_clear(qm); - qm->debug.curr_qm_qp_num = 0; - hpre_show_last_regs_uninit(qm); - hisi_qm_dev_err_uninit(qm); - } - + hpre_probe_uninit(qm); hisi_qm_uninit(qm); } diff --git a/drivers/crypto/hisilicon/migration/acc_vf_migration.c b/drivers/crypto/hisilicon/migration/acc_vf_migration.c index c486dab90dd48d23619b76b214b604210ae16e6b..836a435c499c4be4df3bb8120c1d73175a17f702 100644 --- a/drivers/crypto/hisilicon/migration/acc_vf_migration.c +++ b/drivers/crypto/hisilicon/migration/acc_vf_migration.c @@ -148,9 +148,6 @@ static int qm_config_set(struct hisi_qm *qm, u8 cmd, dma_addr_t dma_addr, { struct qm_mailbox mailbox; - dev_dbg(&qm->pdev->dev, "QM mailbox request to q%u: %u-0x%llx\n", - queue, cmd, (unsigned long long)dma_addr); - qm_mb_pre_init(&mailbox, cmd, dma_addr, queue, op); return qm_mb(qm, &mailbox); diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index bd19a87465752b424dab1cd816e862c4bea0e1a2..8873527a216ec657e96b4ad24595c572d3e08635 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -33,7 +33,7 @@ #define QM_MB_CMD_DATA_SHIFT 32 #define QM_MB_CMD_DATA_MASK GENMASK(31, 0) #define QM_MB_STATUS_MASK GENMASK(12, 9) -#define QM_MB_MAX_WAIT_CNT 6000 +#define QM_MB_MAX_WAIT_CNT 20000 #define QM_MB_WAIT_READY_CNT 10 /* sqc shift */ @@ -313,6 +313,7 @@ static const struct hisi_qm_cap_info qm_cap_info_comm[] = { {QM_SUPPORT_DB_ISOLATION, 0x30, 0, BIT(0), 0x0, 0x0, 0x0}, {QM_SUPPORT_FUNC_QOS, 0x3100, 0, BIT(8), 0x0, 0x0, 0x1}, {QM_SUPPORT_STOP_QP, 0x3100, 0, BIT(9), 0x0, 0x0, 0x1}, + {QM_SUPPORT_STOP_FUNC, 0x3100, 0, BIT(10), 0x0, 0x0, 0x1}, {QM_SUPPORT_MB_COMMAND, 0x3100, 0, BIT(11), 0x0, 0x0, 0x1}, {QM_SUPPORT_SVA_PREFETCH, 0x3100, 0, BIT(14), 0x0, 0x0, 0x1}, }; @@ -444,6 +445,7 @@ static struct qm_typical_qos_table shaper_cbs_s[] = { }; static void qm_irqs_unregister(struct hisi_qm *qm); +static int qm_reset_device(struct hisi_qm *qm); static u32 qm_get_hw_error_status(struct hisi_qm *qm) { @@ -638,9 +640,6 @@ static int hisi_qm_mb_write(struct hisi_qm *qm, u8 cmd, dma_addr_t dma_addr, u16 struct qm_mailbox mailbox; int ret; - dev_dbg(&qm->pdev->dev, "QM mailbox request to q%u: %u-0x%llx\n", - queue, cmd, (unsigned long long)dma_addr); - /* No need to judge if master OOO is blocked. */ if (qm_check_dev_error(pf_qm)) { dev_err(&qm->pdev->dev, "QM mailbox operation failed since qm is stop!\n"); @@ -838,6 +837,12 @@ int hisi_qm_set_algs(struct hisi_qm *qm, u64 alg_msk, const struct qm_dev_alg *d if (!qm->use_uacce) return 0; + if (dev_algs_size >= QM_DEV_ALG_MAX_LEN) { + dev_err(dev, "algs size %u is equal or larger than %d.\n", + dev_algs_size, QM_DEV_ALG_MAX_LEN); + return -EINVAL; + } + algs = devm_kzalloc(dev, QM_DEV_ALG_MAX_LEN * sizeof(char), GFP_KERNEL); if (!algs) return -ENOMEM; @@ -847,10 +852,10 @@ int hisi_qm_set_algs(struct hisi_qm *qm, u64 alg_msk, const struct qm_dev_alg *d strcat(algs, dev_algs[i].alg); ptr = strrchr(algs, '\n'); - if (ptr) + if (ptr) { *ptr = '\0'; - - qm->uacce->algs = algs; + qm->uacce->algs = algs; + } return 0; } @@ -1753,6 +1758,11 @@ static int qm_ping_pf(struct hisi_qm *qm, u64 cmd) return ret; } +static int qm_drain_qm(struct hisi_qm *qm) +{ + return hisi_qm_mb_write(qm, QM_MB_CMD_FLUSH_QM, 0, 0, 0); +} + static int qm_stop_qp(struct hisi_qp *qp) { return hisi_qm_mb_write(qp->qm, QM_MB_CMD_STOP_QP, 0, qp->qp_id, 0); @@ -1921,7 +1931,6 @@ static struct hisi_qp *qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type) qp->event_cb = NULL; qp->req_cb = NULL; - qp->qp_id = qp_id; qp->alg_type = alg_type; qp->is_in_kernel = true; qm->qp_in_used++; @@ -1988,15 +1997,15 @@ static int qm_sq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) sqc.dw3 = cpu_to_le32(QM_MK_SQC_DW3_V2(qm->sqe_size, qp->sq_depth)); sqc.w8 = 0; /* rand_qc */ } - sqc.cq_num = cpu_to_le16(qp_id); sqc.w13 = cpu_to_le16(QM_MK_SQC_W13(0, 1, qp->alg_type)); sqc.base_l = cpu_to_le32(lower_32_bits(qp->sqe_dma)); sqc.base_h = cpu_to_le32(upper_32_bits(qp->sqe_dma)); + sqc.cq_num = cpu_to_le16(qp_id); sqc.pasid = cpu_to_le16(pasid); if (ver >= QM_HW_V3 && qm->use_sva && !qp->is_in_kernel) sqc.w11 = cpu_to_le16(QM_QC_PASID_ENABLE << - QM_QC_PASID_ENABLE_SHIFT); + QM_QC_PASID_ENABLE_SHIFT); return qm_set_and_get_xqc(qm, QM_MB_CMD_SQC, &sqc, qp_id, 0); } @@ -2007,12 +2016,8 @@ static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) enum qm_hw_ver ver = qm->ver; struct qm_cqc cqc = {0}; - cqc.base_l = cpu_to_le32(lower_32_bits(qp->cqe_dma)); - cqc.base_h = cpu_to_le32(upper_32_bits(qp->cqe_dma)); - cqc.pasid = cpu_to_le16(pasid); if (ver == QM_HW_V1) { - cqc.dw3 = cpu_to_le32(QM_MK_CQC_DW3_V1(0, 0, 0, - QM_QC_CQE_SIZE)); + cqc.dw3 = cpu_to_le32(QM_MK_CQC_DW3_V1(0, 0, 0, QM_QC_CQE_SIZE)); cqc.w8 = cpu_to_le16(qp->cq_depth - 1); } else { cqc.dw3 = cpu_to_le32(QM_MK_CQC_DW3_V2(QM_QC_CQE_SIZE, qp->cq_depth)); @@ -2194,9 +2199,11 @@ static int qm_drain_qp(struct hisi_qp *qp) return ret; } + static int qm_stop_qp_nolock(struct hisi_qp *qp) { - struct device *dev = &qp->qm->pdev->dev; + struct hisi_qm *qm = qp->qm; + struct device *dev = &qm->pdev->dev; int ret; /* @@ -2212,11 +2219,15 @@ static int qm_stop_qp_nolock(struct hisi_qp *qp) atomic_set(&qp->qp_status.flags, QP_STOP); - ret = qm_drain_qp(qp); - if (ret) - dev_err(dev, "Failed to drain out data for stopping!\n"); + /* V3 supports direct stop function when FLR prepare */ + if (qm->ver < QM_HW_V3 || qm->status.stop_reason == QM_NORMAL) { + ret = qm_drain_qp(qp); + if (ret) + dev_err(dev, "Failed to drain out data for stopping qp(%u)!\n", + qp->qp_id); + } - flush_workqueue(qp->qm->wq); + flush_workqueue(qm->wq); if (unlikely(qp->is_resetting && atomic_read(&qp->qp_status.used))) qp_stop_fail_cb(qp); @@ -2546,7 +2557,6 @@ static void qm_uacce_base_init(struct hisi_qm *qm) uacce->is_vf = pdev->is_virtfn; uacce->priv = qm; - uacce->algs = qm->algs; uacce->parent = &pdev->dev; qm_get_xqc_depth(qm, &sq_depth, &cq_depth, QM_QP_DEPTH_CAP); @@ -2836,10 +2846,8 @@ static void hisi_qp_memory_uninit(struct hisi_qm *qm, int num) for (i = num - 1; i >= 0; i--) { qdma = &qm->qp_array[i].qdma; dma_free_coherent(dev, qdma->size, qdma->va, qdma->dma); - kfree(qm->poll_data[i].qp_finish_id); } - kfree(qm->poll_data); kfree(qm->qp_array); } @@ -2849,18 +2857,12 @@ static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id, struct device *dev = &qm->pdev->dev; size_t off = qm->sqe_size * sq_depth; struct hisi_qp *qp; - int ret = -ENOMEM; - - qm->poll_data[id].qp_finish_id = kcalloc(qm->qp_num, sizeof(u16), - GFP_KERNEL); - if (!qm->poll_data[id].qp_finish_id) - return -ENOMEM; qp = &qm->qp_array[id]; qp->qdma.va = dma_alloc_coherent(dev, dma_size, &qp->qdma.dma, GFP_KERNEL); if (!qp->qdma.va) - goto err_free_qp_finish_id; + return -ENOMEM; qp->sqe = qp->qdma.va; qp->sqe_dma = qp->qdma.dma; @@ -2873,10 +2875,6 @@ static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id, qp->qp_id = id; return 0; - -err_free_qp_finish_id: - kfree(qm->poll_data[id].qp_finish_id); - return ret; } static inline bool is_iommu_used(struct device *dev) @@ -2982,11 +2980,6 @@ static void hisi_qm_set_state(struct hisi_qm *qm, u8 state) writel(state, qm->io_base + QM_VF_STATE); } -static void hisi_qm_unint_work(struct hisi_qm *qm) -{ - destroy_workqueue(qm->wq); -} - static void hisi_qm_free_rsv_buf(struct hisi_qm *qm) { struct qm_dma *xqc_dma = &qm->xqc_buf.qcdma; @@ -3013,6 +3006,44 @@ static void hisi_qm_memory_uninit(struct hisi_qm *qm) kfree(qm->factor); } +static int qm_clear_device(struct hisi_qm *qm) +{ + acpi_handle handle = ACPI_HANDLE(&qm->pdev->dev); + u32 val; + int ret; + + if (qm->fun_type == QM_HW_VF) + return 0; + + /* Device does not support reset, return */ + if (!qm->err_ini->err_info_init) + return 0; + + qm->err_ini->err_info_init(qm); + + if (!handle) + return 0; + + /* No reset method, return */ + if (!acpi_has_method(handle, qm->err_info.acpi_rst)) + return 0; + + /* OOO register set and check */ + writel(ACC_MASTER_GLOBAL_CTRL_SHUTDOWN, qm->io_base + ACC_MASTER_GLOBAL_CTRL); + + ret = readl_relaxed_poll_timeout(qm->io_base + ACC_MASTER_TRANS_RETURN, + val, + (val == ACC_MASTER_TRANS_RETURN_RW), + POLL_PERIOD, POLL_TIMEOUT); + if (ret) { + pci_warn(qm->pdev, "Device is busy, not clear device.\n"); + writel(0x0, qm->io_base + ACC_MASTER_GLOBAL_CTRL); + return ret; + } + + return qm_reset_device(qm); +} + /** * hisi_qm_uninit() - Uninitialize qm. * @qm: The qm needed uninit. @@ -3021,9 +3052,6 @@ static void hisi_qm_memory_uninit(struct hisi_qm *qm) */ void hisi_qm_uninit(struct hisi_qm *qm) { - qm_cmd_uninit(qm); - hisi_qm_unint_work(qm); - down_write(&qm->qps_lock); hisi_qm_memory_uninit(qm); hisi_qm_set_state(qm, QM_NOT_READY); @@ -3305,16 +3333,29 @@ int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r) down_write(&qm->qps_lock); - qm->status.stop_reason = r; if (atomic_read(&qm->status.flags) == QM_STOP) goto err_unlock; /* Stop all the request sending at first. */ atomic_set(&qm->status.flags, QM_STOP); + qm->status.stop_reason = r; - if (qm->status.stop_reason == QM_SOFT_RESET || - qm->status.stop_reason == QM_DOWN) { + if (qm->status.stop_reason != QM_NORMAL) { hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET); + /* + * When performing soft reset, the hardware will no longer + * do tasks, and the tasks in the device will be flushed + * out directly since the master ooo is closed. + */ + if (test_bit(QM_SUPPORT_STOP_FUNC, &qm->caps) && + r != QM_SOFT_RESET) { + ret = qm_drain_qm(qm); + if (ret) { + dev_err(dev, "failed to drain qm!\n"); + goto err_unlock; + } + } + ret = qm_stop_started_qp(qm); if (ret < 0) { dev_err(dev, "Failed to stop started qp!\n"); @@ -3334,6 +3375,7 @@ int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r) } qm_clear_queues(qm); + qm->status.stop_reason = QM_NORMAL; err_unlock: up_write(&qm->qps_lock); @@ -3372,12 +3414,12 @@ static enum acc_err_result qm_hw_error_handle(struct hisi_qm *qm, bool need_rese } /** - * hisi_qm_dev_err_init() - Initialize device error configuration. + * qm_dev_err_init() - Initialize device error configuration. * @qm: The qm for which we want to do error initialization. * * Initialize QM and device error related configuration. */ -void hisi_qm_dev_err_init(struct hisi_qm *qm) +static void qm_dev_err_init(struct hisi_qm *qm) { if (qm->fun_type == QM_HW_VF) return; @@ -3390,15 +3432,14 @@ void hisi_qm_dev_err_init(struct hisi_qm *qm) } qm->err_ini->hw_err_enable(qm); } -EXPORT_SYMBOL_GPL(hisi_qm_dev_err_init); /** - * hisi_qm_dev_err_uninit() - Uninitialize device error configuration. + * qm_dev_err_uninit() - Uninitialize device error configuration. * @qm: The qm for which we want to do error uninitialization. * * Uninitialize QM and device error related configuration. */ -void hisi_qm_dev_err_uninit(struct hisi_qm *qm) +static void qm_dev_err_uninit(struct hisi_qm *qm) { if (qm->fun_type == QM_HW_VF) return; @@ -3411,7 +3452,6 @@ void hisi_qm_dev_err_uninit(struct hisi_qm *qm) } qm->err_ini->hw_err_disable(qm); } -EXPORT_SYMBOL_GPL(hisi_qm_dev_err_uninit); /** * hisi_qm_free_qps() - free multiple queue pairs. @@ -4283,7 +4323,7 @@ static void qm_dev_ecc_mbit_handle(struct hisi_qm *qm) } } -static int qm_soft_reset(struct hisi_qm *qm) +static int qm_soft_reset_prepare(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; int ret; @@ -4328,10 +4368,15 @@ static int qm_soft_reset(struct hisi_qm *qm) qm->err_ini->close_sva_prefetch(qm); ret = qm_set_pf_mse(qm, false); - if (ret) { + if (ret) pci_err(pdev, "Fails to disable pf MSE bit.\n"); - return ret; - } + + return ret; +} + +static int qm_reset_device(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; /* The reset related sub-control registers are not in PCI BAR */ if (ACPI_HANDLE(&pdev->dev)) { @@ -4350,12 +4395,23 @@ static int qm_soft_reset(struct hisi_qm *qm) pci_err(pdev, "Reset step %llu failed!\n", value); return -EIO; } - } else { - pci_err(pdev, "No reset method!\n"); - return -EINVAL; + + return 0; } - return 0; + pci_err(pdev, "No reset method!\n"); + return -EINVAL; +} + +static int qm_soft_reset(struct hisi_qm *qm) +{ + int ret; + + ret = qm_soft_reset_prepare(qm); + if (ret) + return ret; + + return qm_reset_device(qm); } static int qm_vf_reset_done(struct hisi_qm *qm) @@ -4505,7 +4561,7 @@ static int qm_controller_reset_done(struct hisi_qm *qm) } qm_restart_prepare(qm); - hisi_qm_dev_err_init(qm); + qm_dev_err_init(qm); if (qm->err_ini->open_axi_master_ooo) qm->err_ini->open_axi_master_ooo(qm); @@ -4619,7 +4675,7 @@ void hisi_qm_reset_prepare(struct pci_dev *pdev) return; } - hisi_qm_dev_err_uninit(pf_qm); + qm_dev_err_uninit(pf_qm); /* * Check whether there is an ECC mbit error, * If it occurs, need to wait for soft reset @@ -4691,7 +4747,7 @@ void hisi_qm_reset_done(struct pci_dev *pdev) } } - hisi_qm_dev_err_init(pf_qm); + qm_dev_err_init(pf_qm); ret = qm_restart(qm); if (ret) { @@ -5016,10 +5072,15 @@ static void qm_unregister_abnormal_irq(struct hisi_qm *qm) if (qm->fun_type == QM_HW_VF) return; + if (!qm->err_ini->err_info_init) + return; + val = qm->cap_tables.qm_cap_table[QM_ABN_IRQ_TYPE_CAP_IDX].cap_val; if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK)) return; + qm_dev_err_uninit(qm); + irq_vector = val & QM_IRQ_VECTOR_MASK; free_irq(pci_irq_vector(pdev, irq_vector), qm); } @@ -5033,16 +5094,26 @@ static int qm_register_abnormal_irq(struct hisi_qm *qm) if (qm->fun_type == QM_HW_VF) return 0; + if (!qm->err_ini->err_info_init) { + dev_info(&qm->pdev->dev, "Device doesnot support error init!\n"); + return 0; + } + val = qm->cap_tables.qm_cap_table[QM_ABN_IRQ_TYPE_CAP_IDX].cap_val; if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK)) return 0; irq_vector = val & QM_IRQ_VECTOR_MASK; ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_abnormal_irq, 0, qm->dev_name, qm); - if (ret) + if (ret) { dev_err(&qm->pdev->dev, "failed to request abnormal irq, ret = %d", ret); + return ret; + } - return ret; + INIT_WORK(&qm->rst_work, hisi_qm_controller_reset); + qm_dev_err_init(qm); + + return 0; } static void qm_unregister_mb_cmd_irq(struct hisi_qm *qm) @@ -5054,6 +5125,8 @@ static void qm_unregister_mb_cmd_irq(struct hisi_qm *qm) if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) return; + qm_cmd_uninit(qm); + irq_vector = val & QM_IRQ_VECTOR_MASK; free_irq(pci_irq_vector(pdev, irq_vector), qm); } @@ -5070,12 +5143,18 @@ static int qm_register_mb_cmd_irq(struct hisi_qm *qm) irq_vector = val & QM_IRQ_VECTOR_MASK; ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_mb_cmd_irq, 0, qm->dev_name, qm); - if (ret) + if (ret) { dev_err(&pdev->dev, "failed to request function communication irq, ret = %d", ret); + return ret; + } - return ret; + INIT_WORK(&qm->cmd_process, qm_cmd_process); + qm_cmd_init(qm); + + return 0; } +/* Disable aeq interrupt by hisi_qm_stop(). */ static void qm_unregister_aeq_irq(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; @@ -5089,6 +5168,7 @@ static void qm_unregister_aeq_irq(struct hisi_qm *qm) free_irq(pci_irq_vector(pdev, irq_vector), qm); } +/* Enable aeq interrupt by hisi_qm_start(). */ static int qm_register_aeq_irq(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; @@ -5103,11 +5183,62 @@ static int qm_register_aeq_irq(struct hisi_qm *qm) ret = request_threaded_irq(pci_irq_vector(pdev, irq_vector), NULL, qm_aeq_thread, IRQF_ONESHOT, qm->dev_name, qm); if (ret) - dev_err(&pdev->dev, "failed to request eq irq, ret = %d", ret); + dev_err(&pdev->dev, "failed to request aeq irq, ret = %d", ret); return ret; } +static void qm_uninit_eq_work(struct hisi_qm *qm) +{ + int i; + + for (i = qm->qp_num - 1; i >= 0; i--) + kfree(qm->poll_data[i].qp_finish_id); + + kfree(qm->poll_data); + destroy_workqueue(qm->wq); +} + +static int qm_init_eq_work(struct hisi_qm *qm) +{ + int ret = -ENOMEM; + u16 eq_depth; + int i; + + qm->poll_data = kcalloc(qm->qp_num, sizeof(struct hisi_qm_poll_data), GFP_KERNEL); + if (!qm->poll_data) + return ret; + + qm_get_xqc_depth(qm, &qm->eq_depth, &qm->aeq_depth, QM_XEQ_DEPTH_CAP); + eq_depth = qm->eq_depth >> 1; + for (i = 0; i < qm->qp_num; i++) { + qm->poll_data[i].qp_finish_id = kcalloc(eq_depth, sizeof(u16), GFP_KERNEL); + if (!qm->poll_data[i].qp_finish_id) + goto free_qp_finish_id; + + INIT_WORK(&qm->poll_data[i].work, qm_work_process); + qm->poll_data[i].qm = qm; + } + + qm->wq = alloc_workqueue("%s", WQ_HIGHPRI | WQ_MEM_RECLAIM | + WQ_UNBOUND, num_online_cpus(), + pci_name(qm->pdev)); + if (!qm->wq) { + dev_err(&qm->pdev->dev, "failed to alloc workqueue!\n"); + goto free_qp_finish_id; + } + + return 0; + +free_qp_finish_id: + for (i = i - 1; i >= 0; i--) + kfree(qm->poll_data[i].qp_finish_id); + + kfree(qm->poll_data); + return ret; +} + +/* Disable eq interrupt by hisi_qm_stop(). */ static void qm_unregister_eq_irq(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; @@ -5117,10 +5248,13 @@ static void qm_unregister_eq_irq(struct hisi_qm *qm) if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) return; + qm_uninit_eq_work(qm); + irq_vector = val & QM_IRQ_VECTOR_MASK; free_irq(pci_irq_vector(pdev, irq_vector), qm); } +/* Enable eq interrupt by hisi_qm_start(). */ static int qm_register_eq_irq(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; @@ -5133,10 +5267,18 @@ static int qm_register_eq_irq(struct hisi_qm *qm) irq_vector = val & QM_IRQ_VECTOR_MASK; ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_eq_irq, 0, qm->dev_name, qm); - if (ret) + if (ret) { dev_err(&pdev->dev, "failed to request eq irq, ret = %d", ret); + return ret; + } - return ret; + ret = qm_init_eq_work(qm); + if (ret) { + free_irq(pci_irq_vector(pdev, irq_vector), qm); + return ret; + } + + return 0; } static void qm_irqs_unregister(struct hisi_qm *qm) @@ -5355,8 +5497,14 @@ static int hisi_qm_pci_init(struct hisi_qm *qm) goto err_get_pci_res; } + ret = qm_clear_device(qm); + if (ret) + goto err_free_vectors; + return 0; +err_free_vectors: + pci_free_irq_vectors(pdev); err_get_pci_res: qm_put_pci_res(qm); err_disable_pcidev: @@ -5364,30 +5512,6 @@ static int hisi_qm_pci_init(struct hisi_qm *qm) return ret; } -static int hisi_qm_init_work(struct hisi_qm *qm) -{ - int i; - - for (i = 0; i < qm->qp_num; i++) - INIT_WORK(&qm->poll_data[i].work, qm_work_process); - - if (qm->fun_type == QM_HW_PF) - INIT_WORK(&qm->rst_work, hisi_qm_controller_reset); - - if (qm->ver > QM_HW_V2) - INIT_WORK(&qm->cmd_process, qm_cmd_process); - - qm->wq = alloc_workqueue("%s", WQ_HIGHPRI | WQ_MEM_RECLAIM | - WQ_UNBOUND, num_online_cpus(), - pci_name(qm->pdev)); - if (!qm->wq) { - pci_err(qm->pdev, "failed to alloc workqueue!\n"); - return -ENOMEM; - } - - return 0; -} - static int hisi_qp_alloc_memory(struct hisi_qm *qm) { struct device *dev = &qm->pdev->dev; @@ -5399,19 +5523,12 @@ static int hisi_qp_alloc_memory(struct hisi_qm *qm) if (!qm->qp_array) return -ENOMEM; - qm->poll_data = kcalloc(qm->qp_num, sizeof(struct hisi_qm_poll_data), GFP_KERNEL); - if (!qm->poll_data) { - kfree(qm->qp_array); - return -ENOMEM; - } - qm_get_xqc_depth(qm, &sq_depth, &cq_depth, QM_QP_DEPTH_CAP); /* one more page for device or qp statuses */ qp_dma_size = qm->sqe_size * sq_depth + sizeof(struct qm_cqe) * cq_depth; qp_dma_size = PAGE_ALIGN(qp_dma_size) + PAGE_SIZE; for (i = 0; i < qm->qp_num; i++) { - qm->poll_data[i].qm = qm; ret = hisi_qp_memory_init(qm, qp_dma_size, i, sq_depth, cq_depth); if (ret) goto err_init_qp_mem; @@ -5480,7 +5597,6 @@ static int hisi_qm_memory_init(struct hisi_qm *qm) } while (0) idr_init(&qm->qp_idr); - qm_get_xqc_depth(qm, &qm->eq_depth, &qm->aeq_depth, QM_XEQ_DEPTH_CAP); qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * qm->eq_depth) + QMC_ALIGN(sizeof(struct qm_aeqe) * qm->aeq_depth) + QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) + @@ -5565,16 +5681,8 @@ int hisi_qm_init(struct hisi_qm *qm) if (ret) goto err_alloc_uacce; - ret = hisi_qm_init_work(qm); - if (ret) - goto err_free_qm_memory; - - qm_cmd_init(qm); - return 0; -err_free_qm_memory: - hisi_qm_memory_uninit(qm); err_alloc_uacce: qm_remove_uacce(qm); err_irq_register: @@ -5712,7 +5820,7 @@ static int qm_rebuild_for_resume(struct hisi_qm *qm) } qm_cmd_init(qm); - hisi_qm_dev_err_init(qm); + qm_dev_err_init(qm); /* Set the doorbell timeout to QM_DB_TIMEOUT_CFG ns. */ writel(QM_DB_TIMEOUT_SET, qm->io_base + QM_DB_TIMEOUT_CFG); qm_disable_clock_gate(qm); diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h index c2d3941f7995e20f420e9e3b96d92fe7470f9e89..8c3217d57d5eba15f2457337b76b4e32cc0c23d1 100644 --- a/drivers/crypto/hisilicon/sec2/sec.h +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -55,7 +55,6 @@ struct sec_req { dma_addr_t in_dma; struct sec_cipher_req c_req; struct sec_aead_req aead_req; - struct list_head backlog_head; int err_type; int req_id; @@ -121,7 +120,6 @@ struct sec_qp_ctx { struct sec_alg_res *res; struct sec_ctx *ctx; spinlock_t req_lock; - struct list_head backlog; struct hisi_acc_sgl_pool *c_in_pool; struct hisi_acc_sgl_pool *c_out_pool; }; diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 4bc7088c02914d3b0890f5fa9dcbc4839eb4d13d..d43889094f91127ab7488724d7343392e8c82daa 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -288,10 +288,10 @@ static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req) ret = hisi_qp_send(qp_ctx->qp, &req->sec_sqe); if (ctx->fake_req_limit <= atomic_read(&qp_ctx->qp->qp_status.used) && !ret) { - list_add_tail(&req->backlog_head, &qp_ctx->backlog); + req->fake_busy = true; + spin_unlock_bh(&qp_ctx->req_lock); atomic64_inc(&ctx->sec->debug.dfx.send_cnt); atomic64_inc(&ctx->sec->debug.dfx.send_busy_cnt); - spin_unlock_bh(&qp_ctx->req_lock); return -EBUSY; } spin_unlock_bh(&qp_ctx->req_lock); @@ -557,7 +557,6 @@ static int sec_create_qp_ctx(struct sec_ctx *ctx, int qp_ctx_id) spin_lock_init(&qp_ctx->req_lock); idr_init(&qp_ctx->req_idr); - INIT_LIST_HEAD(&qp_ctx->backlog); ret = sec_alloc_qp_ctx_resource(ctx, qp_ctx); if (ret) @@ -1149,7 +1148,8 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, return 0; } - if (crypto_authenc_extractkeys(&keys, key, keylen)) + ret = crypto_authenc_extractkeys(&keys, key, keylen); + if (ret) goto bad_key; ret = sec_aead_aes_set_key(c_ctx, &keys); @@ -1166,6 +1166,7 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, if ((ctx->a_ctx.mac_len & SEC_SQE_LEN_RATE_MASK) || (ctx->a_ctx.a_key_len & SEC_SQE_LEN_RATE_MASK)) { + ret = -EINVAL; dev_err(dev, "MAC or AUTH key length error!\n"); goto bad_key; } @@ -1174,7 +1175,7 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, bad_key: memzero_explicit(&keys, sizeof(struct crypto_authenc_keys)); - return -EINVAL; + return ret; } @@ -1397,31 +1398,10 @@ static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type) } } -static struct sec_req *sec_back_req_clear(struct sec_ctx *ctx, - struct sec_qp_ctx *qp_ctx) -{ - struct sec_req *backlog_req = NULL; - - spin_lock_bh(&qp_ctx->req_lock); - if (ctx->fake_req_limit >= - atomic_read(&qp_ctx->qp->qp_status.used) && - !list_empty(&qp_ctx->backlog)) { - backlog_req = list_first_entry(&qp_ctx->backlog, - typeof(*backlog_req), backlog_head); - list_del(&backlog_req->backlog_head); - } - spin_unlock_bh(&qp_ctx->req_lock); - - return backlog_req; -} - static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req, int err) { struct skcipher_request *sk_req = req->c_req.sk_req; - struct sec_qp_ctx *qp_ctx = req->qp_ctx; - struct skcipher_request *backlog_sk_req; - struct sec_req *backlog_req; sec_free_req_id(req); @@ -1430,14 +1410,8 @@ static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req, ctx->c_ctx.c_mode == SEC_CMODE_CTR) && req->c_req.encrypt) sec_update_iv(req, SEC_SKCIPHER); - while (1) { - backlog_req = sec_back_req_clear(ctx, qp_ctx); - if (!backlog_req) - break; - - backlog_sk_req = backlog_req->c_req.sk_req; - backlog_sk_req->base.complete(&backlog_sk_req->base, - -EINPROGRESS); + if (req->fake_busy) { + sk_req->base.complete(&sk_req->base, -EINPROGRESS); atomic64_inc(&ctx->sec->debug.dfx.recv_busy_cnt); } @@ -1683,9 +1657,6 @@ static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err) struct sec_aead_req *aead_req = &req->aead_req; struct sec_cipher_req *c_req = &req->c_req; size_t authsize = crypto_aead_authsize(tfm); - struct sec_qp_ctx *qp_ctx = req->qp_ctx; - struct aead_request *backlog_aead_req; - struct sec_req *backlog_req; size_t sz; if (!err && c->c_ctx.c_mode == SEC_CMODE_CBC && c_req->encrypt) @@ -1707,14 +1678,8 @@ static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err) sec_free_req_id(req); - while (1) { - backlog_req = sec_back_req_clear(c, qp_ctx); - if (!backlog_req) - break; - - backlog_aead_req = backlog_req->aead_req.aead_req; - backlog_aead_req->base.complete(&backlog_aead_req->base, - -EINPROGRESS); + if (req->fake_busy) { + a_req->base.complete(&a_req->base, -EINPROGRESS); atomic64_inc(&c->sec->debug.dfx.recv_busy_cnt); } @@ -2102,6 +2067,7 @@ static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) req->c_req.sk_req = sk_req; req->c_req.encrypt = encrypt; req->ctx = ctx; + req->fake_busy = false; ret = sec_skcipher_param_check(ctx, req); if (unlikely(ret)) @@ -2383,6 +2349,7 @@ static int sec_aead_crypto(struct aead_request *a_req, bool encrypt) req->aead_req.aead_req = a_req; req->c_req.encrypt = encrypt; req->ctx = ctx; + req->fake_busy = false; ret = sec_aead_param_check(ctx, req); if (unlikely(ret)) { diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 6b5e6f6d42edd1faf0ed94c4e48b97794824cba8..024c574f4c1cdd0055fb96cfe74badee94832811 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -1083,15 +1083,11 @@ static int sec_pf_probe_init(struct sec_dev *sec) struct hisi_qm *qm = &sec->qm; int ret; - qm->err_ini = &sec_err_ini; - qm->err_ini->err_info_init(qm); - ret = sec_set_user_domain_and_cache(qm); if (ret) return ret; sec_open_sva_prefetch(qm); - hisi_qm_dev_err_init(qm); sec_debug_regs_clear(qm); ret = sec_show_last_regs_init(qm); if (ret) @@ -1140,6 +1136,7 @@ static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) qm->qp_num = pf_q_num; qm->debug.curr_qm_qp_num = pf_q_num; qm->qm_list = &sec_devices; + qm->err_ini = &sec_err_ini; if (pf_q_num_flag) set_bit(QM_MODULE_PARAM, &qm->misc_ctl); } else if (qm->fun_type == QM_HW_VF && qm->ver == QM_HW_V1) { @@ -1204,7 +1201,12 @@ static int sec_probe_init(struct sec_dev *sec) static void sec_probe_uninit(struct hisi_qm *qm) { - hisi_qm_dev_err_uninit(qm); + if (qm->fun_type == QM_HW_VF) + return; + + sec_debug_regs_clear(qm); + sec_show_last_regs_uninit(qm); + sec_close_sva_prefetch(qm); } static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) @@ -1218,6 +1220,7 @@ static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) return -ENOMEM; qm = &sec->qm; + set_bit(QM_DRIVER_DOWN, &qm->misc_ctl); ret = sec_qm_init(qm, pdev); if (ret) { pci_err(pdev, "Failed to init SEC QM (%d)!\n", ret); @@ -1238,6 +1241,9 @@ static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err_probe_uninit; } + /* Device is enabled, clear the flag. */ + clear_bit(QM_DRIVER_DOWN, &qm->misc_ctl); + ret = sec_debugfs_init(qm); if (ret) pci_warn(pdev, "Failed to init debugfs!\n"); @@ -1271,10 +1277,10 @@ static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) hisi_qm_alg_unregister(qm, &sec_devices, ctx_q_num); err_qm_del_list: hisi_qm_del_list(qm, &sec_devices); + hisi_qm_wait_task_finish(qm, &sec_devices); sec_debugfs_exit(qm); hisi_qm_stop(qm, QM_NORMAL); err_probe_uninit: - sec_show_last_regs_uninit(qm); sec_probe_uninit(qm); err_qm_uninit: sec_qm_uninit(qm); @@ -1296,11 +1302,6 @@ static void sec_remove(struct pci_dev *pdev) sec_debugfs_exit(qm); (void)hisi_qm_stop(qm, QM_NORMAL); - - if (qm->fun_type == QM_HW_PF) - sec_debug_regs_clear(qm); - sec_show_last_regs_uninit(qm); - sec_probe_uninit(qm); sec_qm_uninit(qm); diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 6bf607668aa45d06a961cc718d8c12bd2be5af71..5e1fe52e9074e96e99f9ea9363b8f67faf3dd3cd 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -1168,8 +1168,6 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) hisi_zip->ctrl = ctrl; ctrl->hisi_zip = hisi_zip; - qm->err_ini = &hisi_zip_err_ini; - qm->err_ini->err_info_init(qm); ret = hisi_zip_set_user_domain_and_cache(qm); if (ret) @@ -1180,7 +1178,6 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) return ret; hisi_zip_open_sva_prefetch(qm); - hisi_qm_dev_err_init(qm); hisi_zip_debug_regs_clear(qm); ret = hisi_zip_show_last_regs_init(qm); @@ -1230,6 +1227,7 @@ static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) qm->qp_num = pf_q_num; qm->debug.curr_qm_qp_num = pf_q_num; qm->qm_list = &zip_devices; + qm->err_ini = &hisi_zip_err_ini; if (pf_q_num_flag) set_bit(QM_MODULE_PARAM, &qm->misc_ctl); } else if (qm->fun_type == QM_HW_VF && qm->ver == QM_HW_V1) { @@ -1296,6 +1294,15 @@ static int hisi_zip_probe_init(struct hisi_zip *hisi_zip) return 0; } +static void hisi_zip_probe_uninit(struct hisi_qm *qm) +{ + if (qm->fun_type == QM_HW_VF) + return; + + hisi_zip_show_last_regs_uninit(qm); + hisi_zip_close_sva_prefetch(qm); +} + static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id) { struct hisi_zip *hisi_zip; @@ -1307,7 +1314,7 @@ static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id) return -ENOMEM; qm = &hisi_zip->qm; - + set_bit(QM_DRIVER_DOWN, &qm->misc_ctl); ret = hisi_zip_qm_init(qm, pdev); if (ret) { pci_err(pdev, "Failed to init ZIP QM (%d)!\n", ret); @@ -1324,6 +1331,9 @@ static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) goto err_dev_err_uninit; + /* Device is enabled, clear the flag. */ + clear_bit(QM_DRIVER_DOWN, &qm->misc_ctl); + ret = hisi_zip_debugfs_init(qm); if (ret) pci_err(pdev, "failed to init debugfs (%d)!\n", ret); @@ -1357,13 +1367,13 @@ static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id) hisi_qm_alg_unregister(qm, &zip_devices, HZIP_CTX_Q_NUM_DEF); err_qm_del_list: + hisi_qm_wait_task_finish(qm, &zip_devices); hisi_qm_del_list(qm, &zip_devices); hisi_zip_debugfs_exit(qm); hisi_qm_stop(qm, QM_NORMAL); err_dev_err_uninit: - hisi_zip_show_last_regs_uninit(qm); - hisi_qm_dev_err_uninit(qm); + hisi_zip_probe_uninit(qm); err_qm_uninit: hisi_zip_qm_uninit(qm); @@ -1385,8 +1395,7 @@ static void hisi_zip_remove(struct pci_dev *pdev) hisi_zip_debugfs_exit(qm); hisi_qm_stop(qm, QM_NORMAL); - hisi_zip_show_last_regs_uninit(qm); - hisi_qm_dev_err_uninit(qm); + hisi_zip_probe_uninit(qm); hisi_zip_qm_uninit(qm); } diff --git a/include/linux/hisi_acc_qm.h b/include/linux/hisi_acc_qm.h index 0cfc46a5eda277371840f01bf1097397cc906f97..75ade7a14bcfa866b1b9f914e54ac86817a8d4d9 100644 --- a/include/linux/hisi_acc_qm.h +++ b/include/linux/hisi_acc_qm.h @@ -44,6 +44,7 @@ #define QM_MB_CMD_CQC_BT 0x5 #define QM_MB_CMD_SQC_VFT_V2 0x6 #define QM_MB_CMD_STOP_QP 0x8 +#define QM_MB_CMD_FLUSH_QM 0x9 #define QM_MB_CMD_SRC 0xc #define QM_MB_CMD_DST 0xd @@ -152,6 +153,7 @@ enum qm_cap_bits { QM_SUPPORT_DB_ISOLATION = 0x0, QM_SUPPORT_FUNC_QOS, QM_SUPPORT_STOP_QP, + QM_SUPPORT_STOP_FUNC, QM_SUPPORT_MB_COMMAND, QM_SUPPORT_SVA_PREFETCH, QM_SUPPORT_RPM, @@ -395,7 +397,6 @@ struct hisi_qm { struct work_struct rst_work; struct work_struct cmd_process; - const char *algs; bool use_uacce; bool use_sva; bool use_iommu; @@ -554,8 +555,6 @@ void hisi_qm_debug_regs_clear(struct hisi_qm *qm); int hisi_qm_sriov_enable(struct pci_dev *pdev, int max_vfs); int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen); int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs); -void hisi_qm_dev_err_init(struct hisi_qm *qm); -void hisi_qm_dev_err_uninit(struct hisi_qm *qm); int hisi_qm_regs_debugfs_init(struct hisi_qm *qm, struct dfx_diff_registers *dregs, u32 reg_len); void hisi_qm_regs_debugfs_uninit(struct hisi_qm *qm, u32 reg_len);