[openEuler-1.0-LTS] fix panic in blk_mq_run_work_fn()

问题栈：
    BUG: kernel NULL pointer dereference, address: 0000000000000018
    PGD 0 P4D 0
    Oops: 0000 [#1] SMP NOPTI
    CPU: 6 PID: 265 Comm: kworker/6:1H Kdump: loaded Tainted: G           O      5.10.0-60.17.0.h43.eulerosv2r11.x86_64 #1
    Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.12.1-0-ga5cab58-20220320_160524-szxrtosci10000 04/01/2014
    Workqueue: kblockd blk_mq_run_work_fn
    RIP: 0010:blk_mq_delay_run_hw_queues+0xb6/0xe0
    RSP: 0018:ffffacc6803d3d88 EFLAGS: 00010246
    RAX: 0000000000000006 RBX: ffff99e2c3d25008 RCX: 00000000ffffffff
    RDX: 0000000000000000 RSI: 0000000000000003 RDI: ffff99e2c911ae18
    RBP: ffffacc6803d3dd8 R08: 0000000000000000 R09: ffff99e2c0901f6c
    R10: 0000000000000018 R11: 0000000000000018 R12: ffff99e2c911ae18
    R13: 0000000000000000 R14: 0000000000000003 R15: ffff99e2c911ae18
    FS:  0000000000000000(0000) GS:ffff99e6bbf00000(0000) knlGS:0000000000000000
    CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
    CR2: 0000000000000018 CR3: 000000007460a006 CR4: 00000000003706e0
    Call Trace:
     __blk_mq_do_dispatch_sched+0x2a7/0x2c0
     ? newidle_balance+0x23e/0x2f0
     __blk_mq_sched_dispatch_requests+0x13f/0x190
     blk_mq_sched_dispatch_requests+0x30/0x60
     __blk_mq_run_hw_queue+0x47/0xd0
     process_one_work+0x1b0/0x350
     worker_thread+0x49/0x300
     ? rescuer_thread+0x3a0/0x3a0
     kthread+0xfe/0x140
     ? kthread_park+0x90/0x90
     ret_from_fork+0x22/0x30

从vmcore中找到request_queue 可以发现 QUEUE_FLAG_DEAD已被置上：

blk_cleanup_queue
 blk_queue_flag_set(QUEUE_FLAG_DEAD, q);

并且blk_mq_tag_set已被清空：

blk_mq_free_tag_set
 set->map[j].mq_map = NULL;
 set->tags = NULL;

问题根因：

virtblk_remove
 del_gendisk
 blk_cleanup_queue
  blk_put_queue -> 引用计数减到0才会异步的调用blk_release_queue
 blk_mq_free_tag_set

blk_release_queue
 cancel_delayed_work_sync -> 此时可能blk_cleanup_queue和blk_mq_free_tag_set已经完成

问题复现：

由于原问题触发条件较多，并且时序要求高，复现概率极低，这里在
blk_mq_run_work_fn() 添加了一个检测，如果queue已经dead，触发BUG_ON()

内核补丁：
```
diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c
index 740dd83853a6..8e25ac59560b 100644
--- a/block/bfq-iosched.c
+++ b/block/bfq-iosched.c
@@ -5068,14 +5068,7 @@ static struct request *bfq_dispatch_rq_from_bfqq(struct bfq_data *bfqd,

static bool bfq_has_work(struct blk_mq_hw_ctx *hctx)
 {
-       struct bfq_data *bfqd = hctx->queue->elevator->elevator_data;
-
-       /*
-        * Avoiding lock: a race on bfqd->queued should cause at
-        * most a call to dispatch for nothing
-        */
-       return !list_empty_careful(&bfqd->dispatch) ||
-               READ_ONCE(bfqd->queued);
+       return true;
 }

static struct request *__bfq_dispatch_request(struct blk_mq_hw_ctx *hctx)
diff --git a/block/blk-core.c b/block/blk-core.c
index 80fa73c419a9..15b7cb3cf000 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -295,6 +295,11 @@ void blk_queue_start_drain(struct request_queue *q)
  */
 void blk_cleanup_queue(struct request_queue *q)
 {
+       static unsigned long ji;
+
+       if (!ji)
+               ji = jiffies;
+
        /* cannot be called from atomic context */
        might_sleep();

@@ -315,6 +320,11 @@ void blk_cleanup_queue(struct request_queue *q)
        blk_freeze_queue(q);

blk_queue_flag_set(QUEUE_FLAG_DEAD, q);
+       if (jiffies > ji + 50 * HZ) {
+               printk("%s: dealy 10s\n", __func__);
+               msleep(10000);
+               printk("%s: dleay done\n", __func__);
+       }

blk_sync_queue(q);
        if (queue_is_mq(q)) {
diff --git a/block/blk-mq.c b/block/blk-mq.c
index ed1869a305c4..7c7ad1855bd1 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -2354,6 +2354,8 @@ static void blk_mq_run_work_fn(struct work_struct *work)

hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work);

+       if (blk_queue_dead(hctx->queue))
+               BUG_ON(1);
        /*
         * If we are stopped, don't run the queue.
```
复现步骤：
echo bfq > /sys/block/sda/queue/scheduler
dd if=/dev/zero of=/dev/sda bs=8k count=1 oflag=direct
echo 1 > /sys/block/sda/device/delete

复现结果：
删盘时触发了新增的BUG_ON
 ------------[ cut here ]------------
 kernel BUG at block/blk-mq.c:2358!
 invalid opcode: 0000 [#1] PREEMPT SMP
 CPU: 1 PID: 243 Comm: kworker/1:1H Not tainted 5.18.0-rc7-next-20220517-dirty 8
 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS ?-20190727_073836-4
 Workqueue: kblockd blk_mq_run_work_fn
 RIP: 0010:blk_mq_run_work_fn+0x4a/0x60
 Code: 74 01 c3 55 48 83 ef 40 48 83 05 d8 35 b9 0b 01 48 89 e5 e8 b8 fe ff ff 0
 RSP: 0018:ffffc9000062fe68 EFLAGS: 00010202
 RAX: 000000002012269a RBX: 0000000000000000 RCX: 0000000000000000
 RDX: 0000000000000001 RSI: ffff888103d5c648 RDI: ffff888103d5c640
 RBP: ffffc9000062fea8 R08: 00646b636f6c626b R09: ffff888103d5c648
 R10: ffff888100afc1f4 R11: fefefefefefefeff R12: ffff888103d5c640
 R13: ffff888fffc72700 R14: ffff888100afc180 R15: ffff888fffc6b380
 FS:  0000000000000000(0000) GS:ffff888fffc40000(0000) knlGS:0000000000000000
 CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
 CR2: 000055a487b44080 CR3: 00000001759e3000 CR4: 00000000000006e0
 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
 Call Trace:
  <TASK>
  ? process_one_work+0x28e/0x660
  worker_thread+0x264/0x640
  ? rescuer_thread+0x550/0x550
  kthread+0x169/0x1d0
  ? kthread_exit+0x50/0x50
  ret_from_fork+0x1f/0x30
  </TASK>
 Modules linked in:
 ---[ end trace 0000000000000000 ]---
 ---[ end trace 0000000000000000 ]---
 RIP: 0010:blk_mq_run_work_fn+0x4a/0x60
 Code: 74 01 c3 55 48 83 ef 40 48 83 05 d8 35 b9 0b 01 48 89 e5 e8 b8 fe ff ff 0
 RSP: 0018:ffffc9000062fe68 EFLAGS: 00010202
 RAX: 000000002012269a RBX: 0000000000000000 RCX: 0000000000000000
 RDX: 0000000000000001 RSI: ffff888103d5c648 RDI: ffff888103d5c640
 RBP: ffffc9000062fea8 R08: 00646b636f6c626b R09: ffff888103d5c648
 R10: ffff888100afc1f4 R11: fefefefefefefeff R12: ffff888103d5c640
 R13: ffff888fffc72700 R14: ffff888100afc180 R15: ffff888fffc6b380
 FS:  0000000000000000(0000) GS:ffff888fffc40000(0000) knlGS:0000000000000000
 CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
 CR2: 000055a487b44080 CR3: 00000001759e3000 CR4: 00000000000006e0
 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
 Kernel panic - not syncing: Fatal exception

修复方案：
保证在blk_cleanup_queue()结束后，不会有任何'hctx->run_work'还在流程中

GVP openEuler/kernel

内容风险标识

评论 (1)

GVPopenEuler/kernel

内容风险标识