【OLK 5.10】修复scsi_end_request并发场景出现ref->data空指针解引用问题

问题现象
问题现象，系统启动出现空指针引用
	[   33.674894][    C3] Hardware name: Huawei RH2288H V3/BC11HGSA0, BIOS 3.31 08/22/2016
	[   33.674900][    C3] RIP: 0010:scsi_end_request+0xeb/0x110
	[   33.674902][    C3] Code: f9 ff 0f b6 43 18 83 e8 20 83 f8 01 0f 87 72 ff ff ff eb 87 49 8b 44 24 18 f0 48 83 28 01 75 c3 49 8b 44 24 18 49 8d 7c 24 10 <48> 8b 40 08 e8 8c 09 54 00 eb ae 49 8b 44 24 18 f0 48 83 00 01 e9
	[   33.674903][    C3] RSP: 0018:ffffbe724653ce68 EFLAGS: 00010046
	[   33.674905][    C3] RAX: 0000000000000000 RBX: ffff9c4c01247700 RCX: dead000000000122
	[   33.674906][    C3] RDX: 0000000000000000 RSI: 0000000000000046 RDI: ffff9c4b8290c488
	[   33.674907][    C3] RBP: 000000000000000a R08: 0000000000004008 R09: 0000000000000001
	[   33.674907][    C3] R10: 000000000000008d R11: 000000005e3c9000 R12: ffff9c4b8290c478
	[   33.674908][    C3] R13: 0000000000000000 R14: ffff9c4b0bf7c000 R15: ffff9c4c01247818
	[   33.674910][    C3] FS:  0000000000000000(0000) GS:ffff9c5a3fd80000(0000) knlGS:0000000000000000
	[   33.674910][    C3] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
	[   33.674911][    C3] CR2: 0000000000000008 CR3: 0000000c9f1e6002 CR4: 00000000001706e0
	[   33.674916][    C3] Call Trace:
	[   33.687571][  T279] ata7: SATA max UDMA/133 abar m2048@0x92501000 port 0x92501200 irq 61
	[   33.697233][    C3]  <IRQ>
	[   33.697237][    C3]  scsi_io_completion+0x78/0x540
	[   33.697248][    C3]  complete_cmd_fusion+0x251/0x4a0 [megaraid_sas]
	[   33.707861][  T279] ata8: SATA max UDMA/133 abar m2048@0x92501000 port 0x92501280 irq 61
	[   33.721495][    C3]  megasas_isr_fusion+0x7c/0x90 [megaraid_sas]
	[   33.721501][    C3]  __handle_irq_event_percpu+0x3f/0x150
	[   33.736324][  T279] ata9: SATA max UDMA/133 abar m2048@0x92501000 port 0x92501300 irq 61
	[   33.750998][    C3]  handle_irq_event_percpu+0x30/0x80
	[   33.751000][    C3]  handle_irq_event+0x39/0x60
	[   33.751002][    C3]  handle_edge_irq+0x87/0x1b0
	[   33.751005][    C3]  asm_call_irq_on_stack+0x12/0x20
	[   33.751009][    C3]  </IRQ>
	[   33.763165][  T279] ata10: SATA max UDMA/133 abar m2048@0x92501000 port 0x92501380 irq 61
	[   33.796922][    C3]  common_interrupt+0xb9/0x130
	[   33.796924][    C3]  asm_common_interrupt+0x1e/0x40
	[   33.796927][    C3] RIP: 0010:cpuidle_enter_state+0xd8/0x380
	[   33.796929][    C3] Code: 49 89 c4 0f 1f 44 00 00 31 ff e8 c3 3d 97 ff 45 84 ff 74 12 9c 58 f6 c4 02 0f 85 57 02 00 00 31 ff e8 cc f9 9d ff fb 45 85 f6 <0f> 88 f0 00 00 00 49 63 d6 4c 2b 24 24 48 89 d0 48 c1 e0 04 48 01
	[   33.905469][  T828] ata2: SATA link down (SStatus 0 SControl 300)
	[   33.914700][    C3] RSP: 0018:ffffbe7246413e88 EFLAGS: 00000206
	[   33.914701][    C3] RAX: ffff9c5a3fdb4a80 RBX: ffff9c4b070b1000 RCX: 000000000000001f
	[   33.914702][    C3] RDX: 00000007d46f4f02 RSI: 000000003575039e RDI: 0000000000000000
	[   33.914703][    C3] RBP: 0000000000000003 R08: 0000000000000002 R09: 0000000000034240
	[   33.914704][    C3] R10: 003ba04480f7c2f2 R11: ffff9c5a3fdb37e4 R12: 00000007d46f4f02
	[   33.914705][    C3] R13: ffffffff9a0d3ea0 R14: 0000000000000003 R15: 0000000000000000
	[   33.914709][    C3]  ? cpuidle_enter_state+0xbd/0x380
	[   33.914711][    C3]  cpuidle_enter+0x29/0x40
	[   33.914717][    C3]  do_idle+0x20b/0x230
	[   34.122777][  T835] ata5: SATA link down (SStatus 0 SControl 300)
	[   34.125980][    C3]  cpu_startup_entry+0x19/0x20
	[   34.125983][    C3]  start_secondary+0x10d/0x150
	[   34.125986][    C3]  secondary_startup_64_no_verify+0xc2/0xcb
	[   34.125990][    C3] Modules linked in:
	[   34.241484][  T830] ata3: SATA link down (SStatus 0 SControl 300)
	[   34.250353][    C3]  ahci libahci libata crc32c_intel megaraid_sas tg3 dm_mirror dm_region_hash dm_log dm_mod
	[   34.250362][    C3] CR2: 0000000000000008
	[   34.464534][    C3] ---[ end trace 98764d0255c15e95 ]---
	[   35.011738][    C3] RIP: 0010:scsi_end_request+0xeb/0x110
	[   35.011816][  T832] ata4: SATA link down (SStatus 0 SControl 300)
	[   35.024403][    C3] Code: f9 ff 0f b6 43 18 83 e8 20 83 f8 01 0f 87 72 ff ff ff eb 87 49 8b 44 24 18 f0 48 83 28 01 75 c3 49 8b 44 24 18 49 8d 7c 24 10 <48> 8b 40 08 e8 8c 09 54 00 eb ae 49 8b 44 24 18 f0 48 83 00 01 e9
	[   35.024404][    C3] RSP: 0018:ffffbe724653ce68 EFLAGS: 00010046
	[   35.024406][    C3] RAX: 0000000000000000 RBX: ffff9c4c01247700 RCX: dead000000000122
	[   35.024407][    C3] RDX: 0000000000000000 RSI: 0000000000000046 RDI: ffff9c4b8290c488
	[   35.024407][    C3] RBP: 000000000000000a R08: 0000000000004008 R09: 0000000000000001
	[   35.024408][    C3] R10: 000000000000008d R11: 000000005e3c9000 R12: ffff9c4b8290c478
	[   35.024409][    C3] R13: 0000000000000000 R14: ffff9c4b0bf7c000 R15: ffff9c4c01247818
	[   35.024410][    C3] FS:  0000000000000000(0000) GS:ffff9c5a3fd80000(0000) knlGS:0000000000000000
	[   35.024411][    C3] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
	[   35.024412][    C3] CR2: 0000000000000008 CR3: 0000000c9f1e6002 CR4: 00000000001706e0
	[   35.024413][    C3] Kernel panic - not syncing: Fatal exception in interrupt
	[   35.037703][    C3] Kernel Offset: 0x17400000 from 0xffffffff81000000 (relocation range: 0xffffffff80000000-0xffffffffbfffffff)
	[   35.794469][    C3] ---[ end Kernel panic - not syncing: Fatal exception in interrupt ]---

问题分析
CPU#0              	 CPU#1               
---------------     	----------------   
			Step1: 下发io，io完成，到scsi_end_request
Step2:删盘
blk_cleanup_queue
  // 睡眠等待q_usage_counter == 0
  blk_freeze_queue
			scsi_end_request
			  percpu_ref_get
			  ...	
			  percpu_ref_put
                          atomic_long_sub_and_test

// 获取q_count等于0，未休眠
	  percpu_ref_exit
		ref->data置空，free										 
                          ref->data->release(ref) -> 触发null引用

问题复现
修改点如下：
（1） blk_mq_freeze_queue_wait中睡眠等待改为while 轮询等待，未改变原理
（2） scsi_end_request中percpu_ref_put展开，增加延时20ms；

diff --git a/block/blk-core.c b/block/blk-core.c
index 109fb2750453..11f90c2ac90f 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -385,6 +385,7 @@ void blk_cleanup_queue(struct request_queue *q)
         * prevent that blk_mq_run_hw_queues() accesses the hardware queues
         * after draining finished.
         */
+       printk("%s %p\n", __func__, q);
        blk_freeze_queue(q);
 
        rq_qos_exit(q);
@@ -415,8 +416,9 @@ void blk_cleanup_queue(struct request_queue *q)
        if (q->elevator)
                blk_mq_sched_free_requests(q);
        mutex_unlock(&q->sysfs_lock);
-
+       printk("exit queue ref\n");
        percpu_ref_exit(&q->q_usage_counter);
+       printk("exit queue ref data %px\n", q->q_usage_counter.data);
 
        /* @q is and will stay empty, shutdown and put */
        blk_put_queue(q);
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 1941ffc4db85..7b680b9b33c7 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -205,7 +205,10 @@ EXPORT_SYMBOL_GPL(blk_freeze_queue_start);
 
 void blk_mq_freeze_queue_wait(struct request_queue *q)
 {
-       wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter));
+       while (!percpu_ref_is_zero(&q->q_usage_counter)) {
+               msleep(1);
+       }
+       // wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter));
 }
 EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait);
 
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index 9352a309fe55..860624eb36b5 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -569,7 +569,7 @@ static bool scsi_end_request(struct request *req, blk_status_t error,
        struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
        struct scsi_device *sdev = cmd->device;
        struct request_queue *q = sdev->request_queue;
-
+       printk("%s %p\n", __func__, q);
        if (blk_update_request(req, error, bytes))
                return true;
 
@@ -608,7 +608,25 @@ static bool scsi_end_request(struct request *req, blk_status_t error,
 
        scsi_run_queue_async(sdev);
 
-       percpu_ref_put(&q->q_usage_counter);
+       {
+               struct percpu_ref *ref = &q->q_usage_counter;
+               unsigned long __percpu *percpu_count;
+
+               rcu_read_lock();
+
+               if (__ref_is_percpu(ref, &percpu_count)) {
+                       trace_printk("percpu ref\n");
+                       this_cpu_sub(*percpu_count, 1);
+               } else if (unlikely(atomic_long_sub_and_test(1, &ref->data->count))) {
+                       trace_printk("last ref\n");
+                       mdelay(20);
+                       ref->data->release(ref);
+               }
+
+               rcu_read_unlock();
+       }
+
+       printk("%s out %p\n", __func__, q);
        return false;
 }

GVP openEuler/kernel

内容风险标识

评论 (1)

GVPopenEuler/kernel

内容风险标识