From f2a3823e2bfa11fad295a9ac9b9e502153ce1c38 Mon Sep 17 00:00:00 2001 From: Guanjun Date: Fri, 12 Aug 2022 15:08:23 +0800 Subject: [PATCH 1/2] anolis: crypto/ycc: optimize the flow of device error ANBZ: #1844 Optimize the flow of device error handling. That includes: - Clear bme in the top half; - Ring error handling, don't regard it as fatal error anymore. Signed-off-by: Guanjun Link: https://gitee.com/anolis/cloud-kernel/pulls/611 Reviewed-by: Zelin Deng --- drivers/crypto/ycc/ycc_isr.c | 111 +++++++++++++++------------------- drivers/crypto/ycc/ycc_ring.c | 89 +++++++++++++++++++++------ drivers/crypto/ycc/ycc_ring.h | 13 +++- 3 files changed, 134 insertions(+), 79 deletions(-) diff --git a/drivers/crypto/ycc/ycc_isr.c b/drivers/crypto/ycc/ycc_isr.c index 6570a415fe55..04920d313545 100644 --- a/drivers/crypto/ycc/ycc_isr.c +++ b/drivers/crypto/ycc/ycc_isr.c @@ -5,14 +5,14 @@ #include #include #include -#include #include #include "ycc_isr.h" #include "ycc_dev.h" #include "ycc_ring.h" -#define MAX_ERROR_RETRY 50000 /* every 100us, 5s in total */ +extern void ycc_clear_cmd_ring(struct ycc_ring *ring); +extern void ycc_clear_resp_ring(struct ycc_ring *ring); static irqreturn_t ycc_resp_isr(int irq, void *data) { @@ -23,19 +23,6 @@ static irqreturn_t ycc_resp_isr(int irq, void *data) return IRQ_HANDLED; } -static inline void ycc_clear_bme_and_wait_pending(struct pci_dev *pdev) -{ - pci_clear_master(pdev); - - if (pci_wait_for_pending_transaction(pdev)) - pr_warn("Failed to pending transaction\n"); -} - -static inline void ycc_set_bme(struct pci_dev *pdev) -{ - pci_set_master(pdev); -} - static int ycc_send_uevent(struct ycc_dev *ydev, const char *event) { char *envp[3]; @@ -61,29 +48,23 @@ static int ycc_send_uevent(struct ycc_dev *ydev, const char *event) static void ycc_fatal_error(struct ycc_dev *ydev) { struct ycc_ring *ring; - u32 pending_cmd; - int retry = MAX_ERROR_RETRY; int i; for (i = 0; i < YCC_RINGPAIR_NUM; i++) { - /* - * First we make sure all ycc rings's prefetched cmds - * have been processed. - * If timeout, regard it as processed - */ - ring = &ydev->rings[i]; + ring = ydev->rings + i; + if (ring->type != KERN_RING) continue; - pending_cmd = YCC_CSR_RD(ring->csr_vaddr, REG_RING_PENDING_CMD); - while (pending_cmd && retry--) - udelay(100); + spin_lock_bh(&ring->lock); + ycc_clear_cmd_ring(ring); + spin_unlock_bh(&ring->lock); - ycc_clear_ring(ring, pending_cmd); + ycc_clear_resp_ring(ring); } /* - * After ring had been cleared, we should notify + * After all rings had been cleared, we should notify * user space that ycc has fatal error */ ycc_send_uevent(ydev, "YCC_STATUS=fatal"); @@ -97,15 +78,12 @@ static void ycc_process_global_err(struct work_struct *work) u32 hclk_err, xclk_err; u32 xclk_ecc_uncor_err_0, xclk_ecc_uncor_err_1; u32 hclk_ecc_uncor_err; - u64 ycc_ring_status; - u32 pending_cmd; - int retry = MAX_ERROR_RETRY; int i; - /* First disable ycc mastering, no new transactions */ - ycc_clear_bme_and_wait_pending(ydev->pdev); + if (pci_wait_for_pending_transaction(ydev->pdev)) + pr_warn("YCC: Failed to pending transaction\n"); - /* Notify user space ycc is in error handling */ + /* Notify user space YCC is in error handling */ ycc_send_uevent(ydev, "YCC_STATUS=stopped"); hclk_err = YCC_CSR_RD(cfg_bar->vaddr, REG_YCC_HCLK_INT_STATUS); @@ -115,41 +93,52 @@ static void ycc_process_global_err(struct work_struct *work) hclk_ecc_uncor_err = YCC_CSR_RD(cfg_bar->vaddr, REG_YCC_HCLK_MEM_ECC_UNCOR); if ((hclk_err & ~(YCC_HCLK_TRNG_ERR)) || xclk_err || hclk_ecc_uncor_err) { - pr_debug("YCC: Got uncorrected error, must be reset\n"); + pr_err("YCC: Got uncorrected error, must be reset\n"); /* - * Fatal error, as ycc cannot be reset in REE, - * clear ring data. + * Fatal error, as YCC cannot be reset in REE, clear ring data. */ return ycc_fatal_error(ydev); } if (xclk_ecc_uncor_err_0 || xclk_ecc_uncor_err_1) { - pr_debug("YCC: Got algorithm ECC error: %x ,%x\n", + pr_err("YCC: Got algorithm ECC error: %x ,%x\n", xclk_ecc_uncor_err_0, xclk_ecc_uncor_err_1); return ycc_fatal_error(ydev); } + /* This has to be queue error. Handling command rings. */ + for (i = 0; i < YCC_RINGPAIR_NUM; i++) { + ring = ydev->rings + i; + + if (ring->type != KERN_RING) + continue; + + ring->status = YCC_CSR_RD(ring->csr_vaddr, REG_RING_STATUS); + if (ring->status) { + pr_err("YCC: Dev: %d, Ring: %d got ring err: %x\n", + ydev->id, ring->ring_id, ring->status); + spin_lock_bh(&ring->lock); + ycc_clear_cmd_ring(ring); + spin_unlock_bh(&ring->lock); + } + } + /* - * This has to be queue error. As response can respond - * any way, just log the error and ignore it + * Give HW a chance to process all pending_cmds + * through recovering transactions. */ + pci_set_master(ydev->pdev); + + /* Handling response rings. */ for (i = 0; i < YCC_RINGPAIR_NUM; i++) { - ring = &ydev->rings[i]; - pending_cmd = YCC_CSR_RD(ring->csr_vaddr, REG_RING_PENDING_CMD); - while (pending_cmd && retry--) - udelay(100); - - /* Regard as fatal error */ - if (!retry) - return ycc_fatal_error(ydev); - - ycc_ring_status = YCC_CSR_RD(ring->csr_vaddr, REG_RING_STATUS); - if (ycc_ring_status) - pr_debug("YCC: Dev:%d, Ring:%d got ring err:%llx\n", - ydev->id, ring->ring_id, ycc_ring_status); + ring = ydev->rings + i; + + if (ring->type != KERN_RING || !ring->status) + continue; + + ycc_clear_resp_ring(ring); } - ycc_set_bme(ydev->pdev); ycc_g_err_unmask(cfg_bar->vaddr); clear_bit(YDEV_STATUS_ERR, &ydev->status); set_bit(YDEV_STATUS_READY, &ydev->status); @@ -161,16 +150,18 @@ static irqreturn_t ycc_g_err_isr(int irq, void *data) struct ycc_dev *ydev = (struct ycc_dev *)data; struct ycc_bar *cfg_bar; + if (test_and_set_bit(YDEV_STATUS_ERR, &ydev->status)) + return IRQ_HANDLED; + /* Mask global errors until it has been processed */ cfg_bar = &ydev->ycc_bars[YCC_SEC_CFG_BAR]; ycc_g_err_mask(cfg_bar->vaddr); - if (test_and_set_bit(YDEV_STATUS_ERR, &ydev->status)) { - ycc_g_err_unmask(cfg_bar->vaddr); - return IRQ_HANDLED; - } clear_bit(YDEV_STATUS_READY, &ydev->status); + /* Disable YCC mastering, no new transactions */ + pci_clear_master(ydev->pdev); + schedule_work(&ydev->work); return IRQ_HANDLED; } @@ -180,10 +171,8 @@ void ycc_resp_work_process(struct work_struct *work) struct ycc_ring *ring = container_of(work, struct ycc_ring, work); ycc_dequeue(ring); - if (ring->ydev->is_polling) { - udelay(100); + if (ring->ydev->is_polling) schedule_work(work); - } } int ycc_enable_msix(struct ycc_dev *ydev) diff --git a/drivers/crypto/ycc/ycc_ring.c b/drivers/crypto/ycc/ycc_ring.c index ae469e57cd00..80554710a0c3 100644 --- a/drivers/crypto/ycc/ycc_ring.c +++ b/drivers/crypto/ycc/ycc_ring.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include #include +#include #include #include #include @@ -349,7 +350,7 @@ void ycc_dev_rings_release(struct ycc_dev *ydev, int user_rings) /* * Check if the command queue is full. */ -static int ycc_ring_full(struct ycc_ring *ring) +static inline bool ycc_ring_full(struct ycc_ring *ring) { return ring->cmd_rd_ptr == (ring->cmd_wr_ptr + 1) % ring->max_desc; } @@ -357,7 +358,7 @@ static int ycc_ring_full(struct ycc_ring *ring) /* * Check if the response queue is empty */ -static int ycc_ring_empty(struct ycc_ring *ring) +static inline bool ycc_ring_empty(struct ycc_ring *ring) { return ring->resp_rd_ptr == ring->resp_wr_ptr; } @@ -380,8 +381,11 @@ static struct ycc_ring *ycc_select_ring(void) for (i = 0; i < YCC_RINGPAIR_NUM; i++) { cur_r = ydev->rings + i; + + /* Ring is not for kernel */ if (cur_r->type != KERN_RING) continue; + if (!base_r) { /* It means ycc is first used */ base_r = cur_r; @@ -458,11 +462,13 @@ int ycc_enqueue(struct ycc_ring *ring, void *cmd) if (!ring || !ring->ydev || !cmd) return -EINVAL; - /* TODO: Will use tasklet to handle cqe in bottom half */ spin_lock_bh(&ring->lock); - if (!test_bit(YDEV_STATUS_READY, &ring->ydev->status)) { - pr_debug("YCC: equeue error, status is not ready\n"); - ret = -EIO; + if (!test_bit(YDEV_STATUS_READY, &ring->ydev->status) || ycc_ring_stopped(ring)) { + pr_debug("YCC: equeue error, device status: %ld, ring stopped: %d\n", + ring->ydev->status, ycc_ring_stopped(ring)); + + /* Fallback to software */ + ret = -EAGAIN; goto out; } @@ -565,6 +571,9 @@ void ycc_dequeue(struct ycc_ring *ring) struct ycc_resp_desc *resp; int cnt = 0; + if (!test_bit(YDEV_STATUS_READY, &ring->ydev->status) || ycc_ring_stopped(ring)) + return; + ring->resp_wr_ptr = YCC_CSR_RD(ring->csr_vaddr, REG_RING_RSP_WR_PTR); while (!ycc_ring_empty(ring)) { resp = (struct ycc_resp_desc *)ring->resp_base_vaddr + @@ -581,26 +590,72 @@ void ycc_dequeue(struct ycc_ring *ring) } /* - * Clear incompletion cmds in command queue. Before been invoked, must ensure that - * 1. device error occurs in ycc internal. - * 2. enqueue has been disabled. So no need to lock. + * Clear incompletion cmds in command queue while rollback cmd_wr_ptr. + * + * Note: Make sure been invoked when error occurs in YCC internal and + * YCC status is not ready. */ -void ycc_clear_ring(struct ycc_ring *ring, u32 pending_cmd) +void ycc_clear_cmd_ring(struct ycc_ring *ring) { struct ycc_cmd_desc *desc = NULL; ring->cmd_rd_ptr = YCC_CSR_RD(ring->csr_vaddr, REG_RING_CMD_RD_PTR); ring->cmd_wr_ptr = YCC_CSR_RD(ring->csr_vaddr, REG_RING_CMD_WR_PTR); - if (ring->cmd_rd_ptr - pending_cmd < 0) - ring->cmd_rd_ptr += ring->max_desc - pending_cmd; - while (ring->cmd_rd_ptr != ring->cmd_wr_ptr) { - desc = (struct ycc_cmd_desc *)ring->cmd_base_vaddr + - ring->cmd_rd_ptr; + desc = (struct ycc_cmd_desc *)ring->cmd_base_vaddr + ring->cmd_rd_ptr; ycc_cancel_cmd(ring, desc); - if (++ring->cmd_rd_ptr == ring->max_desc) - ring->cmd_rd_ptr = 0; + if (--ring->cmd_wr_ptr == 0) + ring->cmd_wr_ptr = ring->max_desc; + } + + YCC_CSR_WR(ring->csr_vaddr, REG_RING_CMD_WR_PTR, ring->cmd_wr_ptr); +} + +/* + * Clear response queue + * + * Note: Make sure been invoked when error occurs in YCC internal and + * YCC status is not ready. + */ +void ycc_clear_resp_ring(struct ycc_ring *ring) +{ + struct ycc_resp_desc *resp; + int retry; + u32 pending_cmd; + + /* + * Check if the ring has been stopped. *stop* means no + * new transactions, No need to wait for pending_cmds + * been processed under this condition. + */ + retry = ycc_ring_stopped(ring) ? 0 : MAX_ERROR_RETRY; + pending_cmd = YCC_CSR_RD(ring->csr_vaddr, REG_RING_PENDING_CMD); + + ring->resp_wr_ptr = YCC_CSR_RD(ring->csr_vaddr, REG_RING_RSP_WR_PTR); + while (!ycc_ring_empty(ring) || (retry && pending_cmd)) { + if (!ycc_ring_empty(ring)) { + resp = (struct ycc_resp_desc *)ring->resp_base_vaddr + + ring->resp_rd_ptr; + resp->state = CMD_CANCELLED; + ycc_handle_resp(ring, resp); + + if (++ring->resp_rd_ptr == ring->max_desc) + ring->resp_rd_ptr = 0; + + YCC_CSR_WR(ring->csr_vaddr, REG_RING_RSP_RD_PTR, ring->resp_rd_ptr); + } else { + udelay(MAX_SLEEP_US_PER_CHECK); + retry--; + } + + pending_cmd = YCC_CSR_RD(ring->csr_vaddr, REG_RING_PENDING_CMD); + ring->resp_wr_ptr = YCC_CSR_RD(ring->csr_vaddr, REG_RING_RSP_WR_PTR); } + + if (!retry && pending_cmd) + ring->type = INVAL_RING; + + ring->status = 0; } diff --git a/drivers/crypto/ycc/ycc_ring.h b/drivers/crypto/ycc/ycc_ring.h index bc8b310f4e2c..85dbccc80bfe 100644 --- a/drivers/crypto/ycc/ycc_ring.h +++ b/drivers/crypto/ycc/ycc_ring.h @@ -1,9 +1,12 @@ // SPDX-License-Identifier: GPL-2.0 #ifndef __YCC_RING_H #define __YCC_RING_H + #include #include +#include "ycc_dev.h" + #define CMD_ILLEGAL 0x15 #define CMD_UNDERATTACK 0x25 #define CMD_INVALID 0x35 @@ -17,6 +20,9 @@ #define CMD_INVALID_CONTENT_U8 0x7f #define CMD_INVALID_CONTENT_U64 0x7f7f7f7f7f7f7f7fULL +#define MAX_SLEEP_US_PER_CHECK 100 /* every 100us to check register */ +#define MAX_ERROR_RETRY 10000 /* 1s in total */ + enum ring_type { FREE_RING, USER_RING, @@ -26,6 +32,7 @@ enum ring_type { struct ycc_ring { u16 ring_id; + u32 status; atomic_t ref_cnt; void __iomem *csr_vaddr; /* config register address */ resource_size_t csr_paddr; @@ -163,9 +170,13 @@ static inline void ycc_ring_put(struct ycc_ring *ring) atomic_dec(&ring->ref_cnt); } +static inline bool ycc_ring_stopped(struct ycc_ring *ring) +{ + return !!(YCC_CSR_RD(ring->csr_vaddr, REG_RING_CFG) & RING_STOP_BIT); +} + int ycc_enqueue(struct ycc_ring *ring, void *cmd); void ycc_dequeue(struct ycc_ring *ring); -void ycc_clear_ring(struct ycc_ring *ring, u32 pending_cmd); struct ycc_ring *ycc_crypto_get_ring(void); void ycc_crypto_free_ring(struct ycc_ring *ring); int ycc_dev_rings_init(struct ycc_dev *ydev, u32 max_desc, int user_rings); -- Gitee From 3eeedbc6b0fad4c10ee02ed1c4689a383630cef8 Mon Sep 17 00:00:00 2001 From: Guanjun Date: Tue, 9 Aug 2022 10:22:42 +0800 Subject: [PATCH 2/2] anolis: configs: set CONFIG_CRYPTO_DEV_YCC to m on arm64 ANBZ: #1844 This sets CONFIG_CRYPTO_DEV_YCC to m on arm64. Signed-off-by: Guanjun Link: https://gitee.com/anolis/cloud-kernel/pulls/611 Reviewed-by: Zelin Deng --- arch/arm64/configs/anolis-debug_defconfig | 1 + arch/arm64/configs/anolis_defconfig | 1 + arch/x86/configs/anolis-debug_defconfig | 1 + arch/x86/configs/anolis_defconfig | 1 + 4 files changed, 4 insertions(+) diff --git a/arch/arm64/configs/anolis-debug_defconfig b/arch/arm64/configs/anolis-debug_defconfig index 2310ed05dc8f..ca00dc5ee2e4 100644 --- a/arch/arm64/configs/anolis-debug_defconfig +++ b/arch/arm64/configs/anolis-debug_defconfig @@ -5962,6 +5962,7 @@ CONFIG_CRYPTO_DEV_CHELSIO=m # CONFIG_CRYPTO_DEV_HISI_ZIP is not set # CONFIG_CRYPTO_DEV_HISI_HPRE is not set # CONFIG_CRYPTO_DEV_AMLOGIC_GXL is not set +CONFIG_CRYPTO_DEV_YCC=m CONFIG_ASYMMETRIC_KEY_TYPE=y CONFIG_ASYMMETRIC_PUBLIC_KEY_SUBTYPE=y # CONFIG_ASYMMETRIC_TPM_KEY_SUBTYPE is not set diff --git a/arch/arm64/configs/anolis_defconfig b/arch/arm64/configs/anolis_defconfig index 60ea8c3f18a7..b69f83357f22 100644 --- a/arch/arm64/configs/anolis_defconfig +++ b/arch/arm64/configs/anolis_defconfig @@ -5979,6 +5979,7 @@ CONFIG_CRYPTO_DEV_CHELSIO=m # CONFIG_CRYPTO_DEV_HISI_ZIP is not set # CONFIG_CRYPTO_DEV_HISI_HPRE is not set # CONFIG_CRYPTO_DEV_AMLOGIC_GXL is not set +CONFIG_CRYPTO_DEV_YCC=m CONFIG_ASYMMETRIC_KEY_TYPE=y CONFIG_ASYMMETRIC_PUBLIC_KEY_SUBTYPE=y # CONFIG_ASYMMETRIC_TPM_KEY_SUBTYPE is not set diff --git a/arch/x86/configs/anolis-debug_defconfig b/arch/x86/configs/anolis-debug_defconfig index 951fce8b7d08..2b30caf97b5b 100644 --- a/arch/x86/configs/anolis-debug_defconfig +++ b/arch/x86/configs/anolis-debug_defconfig @@ -6598,6 +6598,7 @@ CONFIG_CRYPTO_DEV_CHELSIO=m # CONFIG_CRYPTO_DEV_VIRTIO is not set # CONFIG_CRYPTO_DEV_SAFEXCEL is not set # CONFIG_CRYPTO_DEV_AMLOGIC_GXL is not set +# CONFIG_CRYPTO_DEV_YCC is not set CONFIG_ASYMMETRIC_KEY_TYPE=y CONFIG_ASYMMETRIC_PUBLIC_KEY_SUBTYPE=y # CONFIG_ASYMMETRIC_TPM_KEY_SUBTYPE is not set diff --git a/arch/x86/configs/anolis_defconfig b/arch/x86/configs/anolis_defconfig index 68c9050bcf9f..55fd36474288 100644 --- a/arch/x86/configs/anolis_defconfig +++ b/arch/x86/configs/anolis_defconfig @@ -6592,6 +6592,7 @@ CONFIG_CRYPTO_DEV_CHELSIO=m # CONFIG_CRYPTO_DEV_VIRTIO is not set # CONFIG_CRYPTO_DEV_SAFEXCEL is not set # CONFIG_CRYPTO_DEV_AMLOGIC_GXL is not set +# CONFIG_CRYPTO_DEV_YCC is not set CONFIG_ASYMMETRIC_KEY_TYPE=y CONFIG_ASYMMETRIC_PUBLIC_KEY_SUBTYPE=y # CONFIG_ASYMMETRIC_TPM_KEY_SUBTYPE is not set -- Gitee