diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c index c3b5930106b288bec70850a90f88f2fdc1ed5a38..aa7a1357c3e163b393f32106399b3014b825c651 100644 --- a/block/blk-mq-debugfs.c +++ b/block/blk-mq-debugfs.c @@ -198,6 +198,7 @@ static const char *const hctx_flag_name[] = { HCTX_FLAG_NAME(NO_SCHED), HCTX_FLAG_NAME(STACKING), HCTX_FLAG_NAME(TAG_HCTX_SHARED), + HCTX_FLAG_NAME(DISABLE_FAIR_TAG_SHARING), }; #undef HCTX_FLAG_NAME diff --git a/block/blk-mq.c b/block/blk-mq.c index 6ab7f360ff2ac711bdadb9b6492dc80cb90759fa..97ecabc0d1075e2b64f5e6616a93c1911a73c5e5 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -3935,6 +3935,35 @@ static void blk_mq_map_swqueue(struct request_queue *q) } } +static void queue_update_fair_tag_sharing(struct request_queue *q) +{ + struct blk_mq_hw_ctx *hctx; + unsigned long i; + bool disabled = q->tag_set->disable_fair_tag_sharing; + + queue_for_each_hw_ctx(q, hctx, i) { + if (disabled) + hctx->flags |= BLK_MQ_F_DISABLE_FAIR_TAG_SHARING; + else + hctx->flags &= ~BLK_MQ_F_DISABLE_FAIR_TAG_SHARING; + } + +} + +void blk_mq_update_fair_tag_sharing(struct blk_mq_tag_set *set) +{ + struct request_queue *q; + + lockdep_assert_held(&set->tag_list_lock); + + list_for_each_entry(q, &set->tag_list, tag_set_list) { + blk_mq_freeze_queue(q); + queue_update_fair_tag_sharing(q); + blk_mq_unfreeze_queue(q); + } +} +EXPORT_SYMBOL_GPL(blk_mq_update_fair_tag_sharing); + /* * Caller needs to ensure that we're either frozen/quiesced, or that * the queue isn't live yet. @@ -3989,6 +4018,7 @@ static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set, { mutex_lock(&set->tag_list_lock); + queue_update_fair_tag_sharing(q); /* * Check to see if we're transitioning to shared (from 1 to 2 queues). */ @@ -4767,6 +4797,9 @@ static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, blk_mq_map_swqueue(q); } + list_for_each_entry(q, &set->tag_list, tag_set_list) + queue_update_fair_tag_sharing(q); + reregister: list_for_each_entry(q, &set->tag_list, tag_set_list) { blk_mq_sysfs_register_hctxs(q); diff --git a/block/blk-mq.h b/block/blk-mq.h index 1743857e0b01d9c94406d74aa923433d86852dd5..8b9aac701035aab753b8baa575368d1779d4fba3 100644 --- a/block/blk-mq.h +++ b/block/blk-mq.h @@ -393,7 +393,8 @@ static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx, { unsigned int depth, users; - if (!hctx || !(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) + if (!hctx || !(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) || + (hctx->flags & BLK_MQ_F_DISABLE_FAIR_TAG_SHARING)) return true; /* diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index c2f647a7c1b050e0862a9bd3b0ad0a2a88ba535b..cf5cd93e0fb93f7a828469ed4a9381b770d82b51 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -1958,6 +1958,7 @@ static const struct blk_mq_ops scsi_mq_ops = { .get_rq_budget_token = scsi_mq_get_rq_budget_token, }; +#define FAIR_TAG_SHARING_THRESHOLD 128 int scsi_mq_setup_tags(struct Scsi_Host *shost) { unsigned int cmd_size, sgl_size; @@ -1978,6 +1979,8 @@ int scsi_mq_setup_tags(struct Scsi_Host *shost) tag_set->nr_hw_queues = shost->nr_hw_queues ? : 1; tag_set->nr_maps = shost->nr_maps ? : 1; tag_set->queue_depth = shost->can_queue; + if (tag_set->queue_depth <= FAIR_TAG_SHARING_THRESHOLD) + tag_set->disable_fair_tag_sharing = true; tag_set->cmd_size = cmd_size; tag_set->numa_node = dev_to_node(shost->dma_dev); tag_set->flags = BLK_MQ_F_SHOULD_MERGE; diff --git a/drivers/scsi/scsi_sysfs.c b/drivers/scsi/scsi_sysfs.c index 24f6eefb68030d39869934e6fbefc9fce11d640b..61602cf8d7a2642a8eecfafffe0c6445d012c39c 100644 --- a/drivers/scsi/scsi_sysfs.c +++ b/drivers/scsi/scsi_sysfs.c @@ -403,6 +403,43 @@ show_nr_hw_queues(struct device *dev, struct device_attribute *attr, char *buf) } static DEVICE_ATTR(nr_hw_queues, S_IRUGO, show_nr_hw_queues, NULL); +static ssize_t +fair_sharing_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct Scsi_Host *shost = class_to_shost(dev); + struct blk_mq_tag_set *tag_set = &shost->tag_set; + + return sprintf(buf, "%d\n", !tag_set->disable_fair_tag_sharing); +} + +static ssize_t +fair_sharing_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct Scsi_Host *shost = class_to_shost(dev); + struct blk_mq_tag_set *tag_set = &shost->tag_set; + bool enable; + int res = kstrtobool(buf, &enable); + + if (res < 0) + return res; + + if (tag_set->disable_fair_tag_sharing == !enable) + return count; + + mutex_lock(&tag_set->tag_list_lock); + if (tag_set->disable_fair_tag_sharing == !enable) + goto out_unlock; + + tag_set->disable_fair_tag_sharing = !enable; + blk_mq_update_fair_tag_sharing(tag_set); + +out_unlock: + mutex_unlock(&tag_set->tag_list_lock); + return count; +} +static DEVICE_ATTR_RW(fair_sharing); + static struct attribute *scsi_sysfs_shost_attrs[] = { &dev_attr_use_blk_mq.attr, &dev_attr_unique_id.attr, @@ -421,6 +458,7 @@ static struct attribute *scsi_sysfs_shost_attrs[] = { &dev_attr_host_reset.attr, &dev_attr_eh_deadline.attr, &dev_attr_nr_hw_queues.attr, + &dev_attr_fair_sharing.attr, NULL }; diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 958ed7e89b301e0fa968ed27dd8ae764d47ab68f..65e95a2c4718ec02673daa6ac14cc5302abec088 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -506,6 +506,7 @@ struct blk_mq_tag_set { int numa_node; unsigned int timeout; unsigned int flags; + bool disable_fair_tag_sharing; void *driver_data; struct blk_mq_tags **tags; @@ -656,6 +657,7 @@ enum { */ BLK_MQ_F_STACKING = 1 << 2, BLK_MQ_F_TAG_HCTX_SHARED = 1 << 3, + BLK_MQ_F_DISABLE_FAIR_TAG_SHARING = 1 << 4, BLK_MQ_F_BLOCKING = 1 << 5, /* Do not allow an I/O scheduler to be configured. */ BLK_MQ_F_NO_SCHED = 1 << 6, @@ -998,6 +1000,7 @@ int blk_rq_append_bio(struct request *rq, struct bio *bio); void blk_execute_rq_nowait(struct request *rq, bool at_head); blk_status_t blk_execute_rq(struct request *rq, bool at_head); bool blk_rq_is_poll(struct request *rq); +void blk_mq_update_fair_tag_sharing(struct blk_mq_tag_set *set); struct req_iterator { struct bvec_iter iter;