diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index e359f841c07bc3df86ae029bf860a7830d4d208e..1ff34199efe7aa8a412774bf5e1a4e3f603f3ae4 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -60,6 +60,11 @@ static bool streams; module_param(streams, bool, 0644); MODULE_PARM_DESC(streams, "turn on support for Streams write directives"); +bool panic_on_double_cqe; +EXPORT_SYMBOL_GPL(panic_on_double_cqe); +module_param(panic_on_double_cqe, bool, 0444); +MODULE_PARM_DESC(panic_on_double_cqe, "crash the kernel to save the scene"); + /* * nvme_wq - hosts nvme related works that are not reset or delete * nvme_reset_wq - hosts nvme reset works diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 6e4d385eea0c47fc92aa8e301c75981808fbd65c..c6425e7f5ba33061fa2089613ac66ada4305c94f 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -26,6 +26,8 @@ extern unsigned int nvme_io_timeout; extern unsigned int admin_timeout; #define ADMIN_TIMEOUT (admin_timeout * HZ) +extern bool panic_on_double_cqe; + #define NVME_DEFAULT_KATO 5 #define NVME_KATO_GRACE 10 @@ -536,12 +538,14 @@ static inline struct request *nvme_find_rq(struct blk_mq_tags *tags, if (unlikely(!rq)) { pr_err("could not locate request for tag %#x\n", tag); + BUG_ON(panic_on_double_cqe); return NULL; } if (unlikely(nvme_genctr_mask(nvme_req(rq)->genctr) != genctr)) { dev_err(nvme_req(rq)->ctrl->device, "request %#x genctr mismatch (got %#x expected %#x)\n", tag, genctr, nvme_genctr_mask(nvme_req(rq)->genctr)); + BUG_ON(panic_on_double_cqe); return NULL; } return rq; diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index e95a458310ab6a139b30badd8288cd10cc19eee4..7c9306bd988b9963ecee88a6b04831b08d94f0ca 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -67,6 +67,12 @@ MODULE_PARM_DESC(sgl_threshold, "Use SGLs when average request segment size is larger or equal to " "this size. Use 0 to disable SGLs."); +static bool invalid_next_sqe; +module_param(invalid_next_sqe, bool, 0444); +MODULE_PARM_DESC(invalid_next_sqe, + "Invalid the next sqe, when target obtain sqe incorrectly, they " + "can discover and hold on for debugging."); + static int io_queue_depth_set(const char *val, const struct kernel_param *kp); static const struct kernel_param_ops io_queue_depth_ops = { .set = io_queue_depth_set, @@ -524,11 +530,23 @@ static inline void nvme_write_sq_db(struct nvme_queue *nvmeq, bool write_sq) static void nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd, bool write_sq) { + void *cur; + struct nvme_command *next; + spin_lock(&nvmeq->sq_lock); - memcpy(nvmeq->sq_cmds + (nvmeq->sq_tail << nvmeq->sqes), - cmd, sizeof(*cmd)); + cur = nvmeq->sq_cmds + (nvmeq->sq_tail << nvmeq->sqes); if (++nvmeq->sq_tail == nvmeq->q_depth) nvmeq->sq_tail = 0; + + if (invalid_next_sqe && nvmeq->qid) { + next = (struct nvme_command *)(nvmeq->sq_cmds + + (nvmeq->sq_tail << nvmeq->sqes)); + next->rw.opcode = 0x01; + next->rw.dptr.prp1 = cpu_to_le64(0x1000); + next->rw.slba = cpu_to_le64(U64_MAX); + } + + memcpy(cur, cmd, sizeof(*cmd)); nvme_write_sq_db(nvmeq, write_sq); spin_unlock(&nvmeq->sq_lock); }