diff --git a/block/blk-mq.c b/block/blk-mq.c index 1d1200afb771957fc58051235d012ce955b94716..b81d4a69056cb745fdf037bd5916010cade738bb 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -23,6 +23,9 @@ #include #include #include +#ifndef __GENKSYMS__ +#include +#endif #include #include #include @@ -1676,6 +1679,8 @@ static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx) static void __blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async, unsigned long msecs) { + int work_cpu; + if (unlikely(blk_mq_hctx_stopped(hctx))) return; @@ -1697,7 +1702,13 @@ static void __blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async, if (!percpu_ref_tryget(&hctx->queue->q_usage_counter)) return; - kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work, + if (enhanced_isolcpus && tick_nohz_full_enabled() && + housekeeping_cpu(raw_smp_processor_id(), HK_FLAG_WQ)) + work_cpu = smp_processor_id(); + else + work_cpu = blk_mq_hctx_next_cpu(hctx); + + kblockd_mod_delayed_work_on(work_cpu, &hctx->run_work, msecs_to_jiffies(msecs)); percpu_ref_put(&hctx->queue->q_usage_counter); } diff --git a/include/linux/sched/isolation.h b/include/linux/sched/isolation.h index cc9f393e2a70603ed16974afeb60aa6569bcd26d..2f93081ad7a030729b9f7c9e8af1bba4fc63ac8c 100644 --- a/include/linux/sched/isolation.h +++ b/include/linux/sched/isolation.h @@ -18,6 +18,7 @@ enum hk_flags { }; #ifdef CONFIG_CPU_ISOLATION +extern bool enhanced_isolcpus; DECLARE_STATIC_KEY_FALSE(housekeeping_overridden); extern int housekeeping_any_cpu(enum hk_flags flags); extern const struct cpumask *housekeeping_cpumask(enum hk_flags flags); @@ -28,6 +29,7 @@ extern void __init housekeeping_init(void); #else +#define enhanced_isolcpus 0 static inline int housekeeping_any_cpu(enum hk_flags flags) { return smp_processor_id(); diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c index 5a6ea03f9882d3bb068f1b10cf2da908fc492000..785ef52011160d902e73f3aa51093134a57df13e 100644 --- a/kernel/sched/isolation.c +++ b/kernel/sched/isolation.c @@ -198,3 +198,11 @@ static int __init housekeeping_isolcpus_setup(char *str) return housekeeping_setup(str, flags); } __setup("isolcpus=", housekeeping_isolcpus_setup); + +bool enhanced_isolcpus; +static int __init enhanced_isolcpus_setup(char *str) +{ + enhanced_isolcpus = true; + return 0; +} +__setup("enhanced_isolcpus", enhanced_isolcpus_setup);