diff --git a/block/blk-mq.c b/block/blk-mq.c index 498a6306cbb582b6c7416c2f6bcca2ce0818836b..0f15ef068f141c21886b123ea3cad0c1cdc5ed85 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -2223,9 +2224,17 @@ static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx) */ void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs) { + int work_cpu; + if (unlikely(blk_mq_hctx_stopped(hctx))) return; - kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work, + + if (enhanced_isolcpus && tick_nohz_full_enabled() && + housekeeping_cpu(raw_smp_processor_id(), HK_TYPE_WQ)) + work_cpu = raw_smp_processor_id(); + else + work_cpu = blk_mq_hctx_next_cpu(hctx); + kblockd_mod_delayed_work_on(work_cpu, &hctx->run_work, msecs_to_jiffies(msecs)); } EXPORT_SYMBOL(blk_mq_delay_run_hw_queue); diff --git a/include/linux/sched/isolation.h b/include/linux/sched/isolation.h index fe1a46f30d2409ff0fe0906bd12af567ae225f0e..3894e74e8dc5188575fb28d27ac446b99f730f6a 100644 --- a/include/linux/sched/isolation.h +++ b/include/linux/sched/isolation.h @@ -19,6 +19,7 @@ enum hk_type { }; #ifdef CONFIG_CPU_ISOLATION +extern bool enhanced_isolcpus; DECLARE_STATIC_KEY_FALSE(housekeeping_overridden); extern int housekeeping_any_cpu(enum hk_type type); extern const struct cpumask *housekeeping_cpumask(enum hk_type type); @@ -29,6 +30,8 @@ extern void __init housekeeping_init(void); #else +#define enhanced_isolcpus 0 + static inline int housekeeping_any_cpu(enum hk_type type) { return smp_processor_id(); diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c index 373d42c707bc5d65d70c18d66e0ffd1621d33f5b..3884c245faf5bf7be6be0041a87d95631ee45756 100644 --- a/kernel/sched/isolation.c +++ b/kernel/sched/isolation.c @@ -239,3 +239,11 @@ static int __init housekeeping_isolcpus_setup(char *str) return housekeeping_setup(str, flags); } __setup("isolcpus=", housekeeping_isolcpus_setup); + +bool enhanced_isolcpus; +static int __init enhanced_isolcpus_setup(char *str) +{ + enhanced_isolcpus = true; + return 0; +} +__setup("enhanced_isolcpus", enhanced_isolcpus_setup);