2017-01-17 14:03:22 +01:00
|
|
|
/*
|
|
|
|
* blk-mq scheduling framework
|
|
|
|
*
|
|
|
|
* Copyright (C) 2016 Jens Axboe
|
|
|
|
*/
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/blk-mq.h>
|
|
|
|
|
|
|
|
#include <trace/events/block.h>
|
|
|
|
|
|
|
|
#include "blk.h"
|
|
|
|
#include "blk-mq.h"
|
2017-05-04 16:24:40 +02:00
|
|
|
#include "blk-mq-debugfs.h"
|
2017-01-17 14:03:22 +01:00
|
|
|
#include "blk-mq-sched.h"
|
|
|
|
#include "blk-mq-tag.h"
|
|
|
|
#include "blk-wbt.h"
|
|
|
|
|
|
|
|
void blk_mq_sched_free_hctx_data(struct request_queue *q,
|
|
|
|
void (*exit)(struct blk_mq_hw_ctx *))
|
|
|
|
{
|
|
|
|
struct blk_mq_hw_ctx *hctx;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
queue_for_each_hw_ctx(q, hctx, i) {
|
|
|
|
if (exit && hctx->sched_data)
|
|
|
|
exit(hctx);
|
|
|
|
kfree(hctx->sched_data);
|
|
|
|
hctx->sched_data = NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(blk_mq_sched_free_hctx_data);
|
|
|
|
|
2017-06-16 18:15:25 +02:00
|
|
|
void blk_mq_sched_assign_ioc(struct request *rq, struct bio *bio)
|
2017-01-17 14:03:22 +01:00
|
|
|
{
|
2017-06-16 18:15:25 +02:00
|
|
|
struct request_queue *q = rq->q;
|
|
|
|
struct io_context *ioc = rq_ioc(bio);
|
2017-01-17 14:03:22 +01:00
|
|
|
struct io_cq *icq;
|
|
|
|
|
|
|
|
spin_lock_irq(q->queue_lock);
|
|
|
|
icq = ioc_lookup_icq(ioc, q);
|
|
|
|
spin_unlock_irq(q->queue_lock);
|
|
|
|
|
|
|
|
if (!icq) {
|
|
|
|
icq = ioc_create_icq(ioc, q, GFP_ATOMIC);
|
|
|
|
if (!icq)
|
|
|
|
return;
|
|
|
|
}
|
2017-06-16 18:15:20 +02:00
|
|
|
get_io_context(icq->ioc);
|
2017-06-16 18:15:25 +02:00
|
|
|
rq->elv.icq = icq;
|
2017-01-17 14:03:22 +01:00
|
|
|
}
|
|
|
|
|
2017-06-21 01:56:13 +02:00
|
|
|
/*
|
|
|
|
* Mark a hardware queue as needing a restart. For shared queues, maintain
|
|
|
|
* a count of how many hardware queues are marked for restart.
|
|
|
|
*/
|
|
|
|
static void blk_mq_sched_mark_restart_hctx(struct blk_mq_hw_ctx *hctx)
|
|
|
|
{
|
|
|
|
if (test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state))
|
|
|
|
return;
|
|
|
|
|
blk-mq: remove synchronize_rcu() from blk_mq_del_queue_tag_set()
We have to remove synchronize_rcu() from blk_queue_cleanup(),
otherwise long delay can be caused during lun probe. For removing
it, we have to avoid to iterate the set->tag_list in IO path, eg,
blk_mq_sched_restart().
This patch reverts 5b79413946d (Revert "blk-mq: don't handle
TAG_SHARED in restart"). Given we have fixed enough IO hang issue,
and there isn't any reason to restart all queues in one tags any more,
see the following reasons:
1) blk-mq core can deal with shared-tags case well via blk_mq_get_driver_tag(),
which can wake up queues waiting for driver tag.
2) SCSI is a bit special because it may return BLK_STS_RESOURCE if queue,
target or host is ready, but SCSI built-in restart can cover all these well,
see scsi_end_request(), queue will be rerun after any request initiated from
this host/target is completed.
In my test on scsi_debug(8 luns), this patch may improve IOPS by 20% ~ 30%
when running I/O on these 8 luns concurrently.
Fixes: 705cda97ee3a ("blk-mq: Make it safe to use RCU to iterate over blk_mq_tag_set.tag_list")
Cc: Omar Sandoval <osandov@fb.com>
Cc: Bart Van Assche <bart.vanassche@wdc.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Martin K. Petersen <martin.petersen@oracle.com>
Cc: linux-scsi@vger.kernel.org
Reported-by: Andrew Jones <drjones@redhat.com>
Tested-by: Andrew Jones <drjones@redhat.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2018-06-25 13:31:48 +02:00
|
|
|
set_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state);
|
2017-06-21 01:56:13 +02:00
|
|
|
}
|
|
|
|
|
blk-mq: remove synchronize_rcu() from blk_mq_del_queue_tag_set()
We have to remove synchronize_rcu() from blk_queue_cleanup(),
otherwise long delay can be caused during lun probe. For removing
it, we have to avoid to iterate the set->tag_list in IO path, eg,
blk_mq_sched_restart().
This patch reverts 5b79413946d (Revert "blk-mq: don't handle
TAG_SHARED in restart"). Given we have fixed enough IO hang issue,
and there isn't any reason to restart all queues in one tags any more,
see the following reasons:
1) blk-mq core can deal with shared-tags case well via blk_mq_get_driver_tag(),
which can wake up queues waiting for driver tag.
2) SCSI is a bit special because it may return BLK_STS_RESOURCE if queue,
target or host is ready, but SCSI built-in restart can cover all these well,
see scsi_end_request(), queue will be rerun after any request initiated from
this host/target is completed.
In my test on scsi_debug(8 luns), this patch may improve IOPS by 20% ~ 30%
when running I/O on these 8 luns concurrently.
Fixes: 705cda97ee3a ("blk-mq: Make it safe to use RCU to iterate over blk_mq_tag_set.tag_list")
Cc: Omar Sandoval <osandov@fb.com>
Cc: Bart Van Assche <bart.vanassche@wdc.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Martin K. Petersen <martin.petersen@oracle.com>
Cc: linux-scsi@vger.kernel.org
Reported-by: Andrew Jones <drjones@redhat.com>
Tested-by: Andrew Jones <drjones@redhat.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2018-06-25 13:31:48 +02:00
|
|
|
void blk_mq_sched_restart(struct blk_mq_hw_ctx *hctx)
|
2017-06-21 01:56:13 +02:00
|
|
|
{
|
|
|
|
if (!test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state))
|
blk-mq: remove synchronize_rcu() from blk_mq_del_queue_tag_set()
We have to remove synchronize_rcu() from blk_queue_cleanup(),
otherwise long delay can be caused during lun probe. For removing
it, we have to avoid to iterate the set->tag_list in IO path, eg,
blk_mq_sched_restart().
This patch reverts 5b79413946d (Revert "blk-mq: don't handle
TAG_SHARED in restart"). Given we have fixed enough IO hang issue,
and there isn't any reason to restart all queues in one tags any more,
see the following reasons:
1) blk-mq core can deal with shared-tags case well via blk_mq_get_driver_tag(),
which can wake up queues waiting for driver tag.
2) SCSI is a bit special because it may return BLK_STS_RESOURCE if queue,
target or host is ready, but SCSI built-in restart can cover all these well,
see scsi_end_request(), queue will be rerun after any request initiated from
this host/target is completed.
In my test on scsi_debug(8 luns), this patch may improve IOPS by 20% ~ 30%
when running I/O on these 8 luns concurrently.
Fixes: 705cda97ee3a ("blk-mq: Make it safe to use RCU to iterate over blk_mq_tag_set.tag_list")
Cc: Omar Sandoval <osandov@fb.com>
Cc: Bart Van Assche <bart.vanassche@wdc.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Martin K. Petersen <martin.petersen@oracle.com>
Cc: linux-scsi@vger.kernel.org
Reported-by: Andrew Jones <drjones@redhat.com>
Tested-by: Andrew Jones <drjones@redhat.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2018-06-25 13:31:48 +02:00
|
|
|
return;
|
|
|
|
clear_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state);
|
2017-06-21 01:56:13 +02:00
|
|
|
|
blk-mq: remove synchronize_rcu() from blk_mq_del_queue_tag_set()
We have to remove synchronize_rcu() from blk_queue_cleanup(),
otherwise long delay can be caused during lun probe. For removing
it, we have to avoid to iterate the set->tag_list in IO path, eg,
blk_mq_sched_restart().
This patch reverts 5b79413946d (Revert "blk-mq: don't handle
TAG_SHARED in restart"). Given we have fixed enough IO hang issue,
and there isn't any reason to restart all queues in one tags any more,
see the following reasons:
1) blk-mq core can deal with shared-tags case well via blk_mq_get_driver_tag(),
which can wake up queues waiting for driver tag.
2) SCSI is a bit special because it may return BLK_STS_RESOURCE if queue,
target or host is ready, but SCSI built-in restart can cover all these well,
see scsi_end_request(), queue will be rerun after any request initiated from
this host/target is completed.
In my test on scsi_debug(8 luns), this patch may improve IOPS by 20% ~ 30%
when running I/O on these 8 luns concurrently.
Fixes: 705cda97ee3a ("blk-mq: Make it safe to use RCU to iterate over blk_mq_tag_set.tag_list")
Cc: Omar Sandoval <osandov@fb.com>
Cc: Bart Van Assche <bart.vanassche@wdc.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Martin K. Petersen <martin.petersen@oracle.com>
Cc: linux-scsi@vger.kernel.org
Reported-by: Andrew Jones <drjones@redhat.com>
Tested-by: Andrew Jones <drjones@redhat.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2018-06-25 13:31:48 +02:00
|
|
|
blk_mq_run_hw_queue(hctx, true);
|
2017-06-21 01:56:13 +02:00
|
|
|
}
|
|
|
|
|
2017-10-27 06:43:30 +02:00
|
|
|
/*
|
|
|
|
* Only SCSI implements .get_budget and .put_budget, and SCSI restarts
|
|
|
|
* its queue by itself in its completion handler, so we don't need to
|
|
|
|
* restart queue if .get_budget() returns BLK_STS_NO_RESOURCE.
|
|
|
|
*/
|
|
|
|
static void blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)
|
2017-10-14 11:22:26 +02:00
|
|
|
{
|
|
|
|
struct request_queue *q = hctx->queue;
|
|
|
|
struct elevator_queue *e = q->elevator;
|
|
|
|
LIST_HEAD(rq_list);
|
|
|
|
|
|
|
|
do {
|
2017-10-14 11:22:29 +02:00
|
|
|
struct request *rq;
|
2017-10-14 11:22:26 +02:00
|
|
|
|
2017-10-14 11:22:29 +02:00
|
|
|
if (e->type->ops.mq.has_work &&
|
|
|
|
!e->type->ops.mq.has_work(hctx))
|
2017-10-14 11:22:26 +02:00
|
|
|
break;
|
2017-10-14 11:22:29 +02:00
|
|
|
|
2017-11-04 19:21:12 +01:00
|
|
|
if (!blk_mq_get_dispatch_budget(hctx))
|
2017-10-27 06:43:30 +02:00
|
|
|
break;
|
2017-10-14 11:22:29 +02:00
|
|
|
|
|
|
|
rq = e->type->ops.mq.dispatch_request(hctx);
|
|
|
|
if (!rq) {
|
|
|
|
blk_mq_put_dispatch_budget(hctx);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Now this rq owns the budget which has to be released
|
|
|
|
* if this rq won't be queued to driver via .queue_rq()
|
|
|
|
* in blk_mq_dispatch_rq_list().
|
|
|
|
*/
|
2017-10-14 11:22:26 +02:00
|
|
|
list_add(&rq->queuelist, &rq_list);
|
2017-10-14 11:22:29 +02:00
|
|
|
} while (blk_mq_dispatch_rq_list(q, &rq_list, true));
|
2017-10-14 11:22:26 +02:00
|
|
|
}
|
|
|
|
|
blk-mq-sched: improve dispatching from sw queue
SCSI devices use host-wide tagset, and the shared driver tag space is
often quite big. However, there is also a queue depth for each lun(
.cmd_per_lun), which is often small, for example, on both lpfc and
qla2xxx, .cmd_per_lun is just 3.
So lots of requests may stay in sw queue, and we always flush all
belonging to same hw queue and dispatch them all to driver.
Unfortunately it is easy to cause queue busy because of the small
.cmd_per_lun. Once these requests are flushed out, they have to stay in
hctx->dispatch, and no bio merge can happen on these requests, and
sequential IO performance is harmed.
This patch introduces blk_mq_dequeue_from_ctx for dequeuing a request
from a sw queue, so that we can dispatch them in scheduler's way. We can
then avoid dequeueing too many requests from sw queue, since we don't
flush ->dispatch completely.
This patch improves dispatching from sw queue by using the .get_budget
and .put_budget callbacks.
Reviewed-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2017-10-14 11:22:30 +02:00
|
|
|
static struct blk_mq_ctx *blk_mq_next_ctx(struct blk_mq_hw_ctx *hctx,
|
|
|
|
struct blk_mq_ctx *ctx)
|
|
|
|
{
|
|
|
|
unsigned idx = ctx->index_hw;
|
|
|
|
|
|
|
|
if (++idx == hctx->nr_ctx)
|
|
|
|
idx = 0;
|
|
|
|
|
|
|
|
return hctx->ctxs[idx];
|
|
|
|
}
|
|
|
|
|
2017-10-27 06:43:30 +02:00
|
|
|
/*
|
|
|
|
* Only SCSI implements .get_budget and .put_budget, and SCSI restarts
|
|
|
|
* its queue by itself in its completion handler, so we don't need to
|
|
|
|
* restart queue if .get_budget() returns BLK_STS_NO_RESOURCE.
|
|
|
|
*/
|
|
|
|
static void blk_mq_do_dispatch_ctx(struct blk_mq_hw_ctx *hctx)
|
blk-mq-sched: improve dispatching from sw queue
SCSI devices use host-wide tagset, and the shared driver tag space is
often quite big. However, there is also a queue depth for each lun(
.cmd_per_lun), which is often small, for example, on both lpfc and
qla2xxx, .cmd_per_lun is just 3.
So lots of requests may stay in sw queue, and we always flush all
belonging to same hw queue and dispatch them all to driver.
Unfortunately it is easy to cause queue busy because of the small
.cmd_per_lun. Once these requests are flushed out, they have to stay in
hctx->dispatch, and no bio merge can happen on these requests, and
sequential IO performance is harmed.
This patch introduces blk_mq_dequeue_from_ctx for dequeuing a request
from a sw queue, so that we can dispatch them in scheduler's way. We can
then avoid dequeueing too many requests from sw queue, since we don't
flush ->dispatch completely.
This patch improves dispatching from sw queue by using the .get_budget
and .put_budget callbacks.
Reviewed-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2017-10-14 11:22:30 +02:00
|
|
|
{
|
|
|
|
struct request_queue *q = hctx->queue;
|
|
|
|
LIST_HEAD(rq_list);
|
|
|
|
struct blk_mq_ctx *ctx = READ_ONCE(hctx->dispatch_from);
|
|
|
|
|
|
|
|
do {
|
|
|
|
struct request *rq;
|
|
|
|
|
|
|
|
if (!sbitmap_any_bit_set(&hctx->ctx_map))
|
|
|
|
break;
|
|
|
|
|
2017-11-04 19:21:12 +01:00
|
|
|
if (!blk_mq_get_dispatch_budget(hctx))
|
2017-10-27 06:43:30 +02:00
|
|
|
break;
|
blk-mq-sched: improve dispatching from sw queue
SCSI devices use host-wide tagset, and the shared driver tag space is
often quite big. However, there is also a queue depth for each lun(
.cmd_per_lun), which is often small, for example, on both lpfc and
qla2xxx, .cmd_per_lun is just 3.
So lots of requests may stay in sw queue, and we always flush all
belonging to same hw queue and dispatch them all to driver.
Unfortunately it is easy to cause queue busy because of the small
.cmd_per_lun. Once these requests are flushed out, they have to stay in
hctx->dispatch, and no bio merge can happen on these requests, and
sequential IO performance is harmed.
This patch introduces blk_mq_dequeue_from_ctx for dequeuing a request
from a sw queue, so that we can dispatch them in scheduler's way. We can
then avoid dequeueing too many requests from sw queue, since we don't
flush ->dispatch completely.
This patch improves dispatching from sw queue by using the .get_budget
and .put_budget callbacks.
Reviewed-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2017-10-14 11:22:30 +02:00
|
|
|
|
|
|
|
rq = blk_mq_dequeue_from_ctx(hctx, ctx);
|
|
|
|
if (!rq) {
|
|
|
|
blk_mq_put_dispatch_budget(hctx);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Now this rq owns the budget which has to be released
|
|
|
|
* if this rq won't be queued to driver via .queue_rq()
|
|
|
|
* in blk_mq_dispatch_rq_list().
|
|
|
|
*/
|
|
|
|
list_add(&rq->queuelist, &rq_list);
|
|
|
|
|
|
|
|
/* round robin for fair dispatch */
|
|
|
|
ctx = blk_mq_next_ctx(hctx, rq->mq_ctx);
|
|
|
|
|
|
|
|
} while (blk_mq_dispatch_rq_list(q, &rq_list, true));
|
|
|
|
|
|
|
|
WRITE_ONCE(hctx->dispatch_from, ctx);
|
|
|
|
}
|
|
|
|
|
2017-10-27 06:43:30 +02:00
|
|
|
void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)
|
2017-01-17 14:03:22 +01:00
|
|
|
{
|
blk-mq: use the right hctx when getting a driver tag fails
While dispatching requests, if we fail to get a driver tag, we mark the
hardware queue as waiting for a tag and put the requests on a
hctx->dispatch list to be run later when a driver tag is freed. However,
blk_mq_dispatch_rq_list() may dispatch requests from multiple hardware
queues if using a single-queue scheduler with a multiqueue device. If
blk_mq_get_driver_tag() fails, it doesn't update the hardware queue we
are processing. This means we end up using the hardware queue of the
previous request, which may or may not be the same as that of the
current request. If it isn't, the wrong hardware queue will end up
waiting for a tag, and the requests will be on the wrong dispatch list,
leading to a hang.
The fix is twofold:
1. Make sure we save which hardware queue we were trying to get a
request for in blk_mq_get_driver_tag() regardless of whether it
succeeds or not.
2. Make blk_mq_dispatch_rq_list() take a request_queue instead of a
blk_mq_hw_queue to make it clear that it must handle multiple
hardware queues, since I've already messed this up on a couple of
occasions.
This didn't appear in testing with nvme and mq-deadline because nvme has
more driver tags than the default number of scheduler tags. However,
with the blk_mq_update_nr_hw_queues() fix, it showed up with nbd.
Signed-off-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2017-04-07 16:56:26 +02:00
|
|
|
struct request_queue *q = hctx->queue;
|
|
|
|
struct elevator_queue *e = q->elevator;
|
2017-02-17 19:39:26 +01:00
|
|
|
const bool has_sched_dispatch = e && e->type->ops.mq.dispatch_request;
|
2017-01-17 14:03:22 +01:00
|
|
|
LIST_HEAD(rq_list);
|
|
|
|
|
2017-06-18 22:24:27 +02:00
|
|
|
/* RCU or SRCU read lock is needed before checking quiesced flag */
|
|
|
|
if (unlikely(blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(q)))
|
2017-10-27 06:43:30 +02:00
|
|
|
return;
|
2017-01-17 14:03:22 +01:00
|
|
|
|
|
|
|
hctx->run++;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If we have previous entries on our dispatch list, grab them first for
|
|
|
|
* more fair dispatch.
|
|
|
|
*/
|
|
|
|
if (!list_empty_careful(&hctx->dispatch)) {
|
|
|
|
spin_lock(&hctx->lock);
|
|
|
|
if (!list_empty(&hctx->dispatch))
|
|
|
|
list_splice_init(&hctx->dispatch, &rq_list);
|
|
|
|
spin_unlock(&hctx->lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Only ask the scheduler for requests, if we didn't have residual
|
|
|
|
* requests from the dispatch list. This is to avoid the case where
|
|
|
|
* we only ever dispatch a fraction of the requests available because
|
|
|
|
* of low device queue depth. Once we pull requests out of the IO
|
|
|
|
* scheduler, we can no longer merge or sort them. So it's best to
|
|
|
|
* leave them there for as long as we can. Mark the hw queue as
|
|
|
|
* needing a restart in that case.
|
2017-10-14 11:22:26 +02:00
|
|
|
*
|
|
|
|
* We want to dispatch from the scheduler if there was nothing
|
|
|
|
* on the dispatch list or we were able to dispatch from the
|
|
|
|
* dispatch list.
|
2017-01-17 14:03:22 +01:00
|
|
|
*/
|
2017-01-26 20:40:07 +01:00
|
|
|
if (!list_empty(&rq_list)) {
|
2017-02-22 19:58:30 +01:00
|
|
|
blk_mq_sched_mark_restart_hctx(hctx);
|
blk-mq-sched: improve dispatching from sw queue
SCSI devices use host-wide tagset, and the shared driver tag space is
often quite big. However, there is also a queue depth for each lun(
.cmd_per_lun), which is often small, for example, on both lpfc and
qla2xxx, .cmd_per_lun is just 3.
So lots of requests may stay in sw queue, and we always flush all
belonging to same hw queue and dispatch them all to driver.
Unfortunately it is easy to cause queue busy because of the small
.cmd_per_lun. Once these requests are flushed out, they have to stay in
hctx->dispatch, and no bio merge can happen on these requests, and
sequential IO performance is harmed.
This patch introduces blk_mq_dequeue_from_ctx for dequeuing a request
from a sw queue, so that we can dispatch them in scheduler's way. We can
then avoid dequeueing too many requests from sw queue, since we don't
flush ->dispatch completely.
This patch improves dispatching from sw queue by using the .get_budget
and .put_budget callbacks.
Reviewed-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2017-10-14 11:22:30 +02:00
|
|
|
if (blk_mq_dispatch_rq_list(q, &rq_list, false)) {
|
|
|
|
if (has_sched_dispatch)
|
2017-10-27 06:43:30 +02:00
|
|
|
blk_mq_do_dispatch_sched(hctx);
|
blk-mq-sched: improve dispatching from sw queue
SCSI devices use host-wide tagset, and the shared driver tag space is
often quite big. However, there is also a queue depth for each lun(
.cmd_per_lun), which is often small, for example, on both lpfc and
qla2xxx, .cmd_per_lun is just 3.
So lots of requests may stay in sw queue, and we always flush all
belonging to same hw queue and dispatch them all to driver.
Unfortunately it is easy to cause queue busy because of the small
.cmd_per_lun. Once these requests are flushed out, they have to stay in
hctx->dispatch, and no bio merge can happen on these requests, and
sequential IO performance is harmed.
This patch introduces blk_mq_dequeue_from_ctx for dequeuing a request
from a sw queue, so that we can dispatch them in scheduler's way. We can
then avoid dequeueing too many requests from sw queue, since we don't
flush ->dispatch completely.
This patch improves dispatching from sw queue by using the .get_budget
and .put_budget callbacks.
Reviewed-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2017-10-14 11:22:30 +02:00
|
|
|
else
|
2017-10-27 06:43:30 +02:00
|
|
|
blk_mq_do_dispatch_ctx(hctx);
|
blk-mq-sched: improve dispatching from sw queue
SCSI devices use host-wide tagset, and the shared driver tag space is
often quite big. However, there is also a queue depth for each lun(
.cmd_per_lun), which is often small, for example, on both lpfc and
qla2xxx, .cmd_per_lun is just 3.
So lots of requests may stay in sw queue, and we always flush all
belonging to same hw queue and dispatch them all to driver.
Unfortunately it is easy to cause queue busy because of the small
.cmd_per_lun. Once these requests are flushed out, they have to stay in
hctx->dispatch, and no bio merge can happen on these requests, and
sequential IO performance is harmed.
This patch introduces blk_mq_dequeue_from_ctx for dequeuing a request
from a sw queue, so that we can dispatch them in scheduler's way. We can
then avoid dequeueing too many requests from sw queue, since we don't
flush ->dispatch completely.
This patch improves dispatching from sw queue by using the .get_budget
and .put_budget callbacks.
Reviewed-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2017-10-14 11:22:30 +02:00
|
|
|
}
|
2017-10-14 11:22:26 +02:00
|
|
|
} else if (has_sched_dispatch) {
|
2017-10-27 06:43:30 +02:00
|
|
|
blk_mq_do_dispatch_sched(hctx);
|
2018-07-03 17:03:16 +02:00
|
|
|
} else if (hctx->dispatch_busy) {
|
|
|
|
/* dequeue request one by one from sw queue if queue is busy */
|
2017-10-27 06:43:30 +02:00
|
|
|
blk_mq_do_dispatch_ctx(hctx);
|
2017-10-14 11:22:26 +02:00
|
|
|
} else {
|
2017-01-26 20:40:07 +01:00
|
|
|
blk_mq_flush_busy_ctxs(hctx, &rq_list);
|
2017-10-14 11:22:29 +02:00
|
|
|
blk_mq_dispatch_rq_list(q, &rq_list, false);
|
2017-02-17 19:39:26 +01:00
|
|
|
}
|
2017-01-17 14:03:22 +01:00
|
|
|
}
|
|
|
|
|
2017-02-03 17:48:28 +01:00
|
|
|
bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,
|
|
|
|
struct request **merged_request)
|
2017-01-17 14:03:22 +01:00
|
|
|
{
|
|
|
|
struct request *rq;
|
|
|
|
|
2017-02-08 14:46:48 +01:00
|
|
|
switch (elv_merge(q, &rq, bio)) {
|
|
|
|
case ELEVATOR_BACK_MERGE:
|
2017-01-17 14:03:22 +01:00
|
|
|
if (!blk_mq_sched_allow_merge(q, rq, bio))
|
|
|
|
return false;
|
2017-02-08 14:46:48 +01:00
|
|
|
if (!bio_attempt_back_merge(q, rq, bio))
|
|
|
|
return false;
|
|
|
|
*merged_request = attempt_back_merge(q, rq);
|
|
|
|
if (!*merged_request)
|
|
|
|
elv_merged_request(q, rq, ELEVATOR_BACK_MERGE);
|
|
|
|
return true;
|
|
|
|
case ELEVATOR_FRONT_MERGE:
|
2017-01-17 14:03:22 +01:00
|
|
|
if (!blk_mq_sched_allow_merge(q, rq, bio))
|
|
|
|
return false;
|
2017-02-08 14:46:48 +01:00
|
|
|
if (!bio_attempt_front_merge(q, rq, bio))
|
|
|
|
return false;
|
|
|
|
*merged_request = attempt_front_merge(q, rq);
|
|
|
|
if (!*merged_request)
|
|
|
|
elv_merged_request(q, rq, ELEVATOR_FRONT_MERGE);
|
|
|
|
return true;
|
2018-02-01 22:41:15 +01:00
|
|
|
case ELEVATOR_DISCARD_MERGE:
|
|
|
|
return bio_attempt_discard_merge(q, rq, bio);
|
2017-02-08 14:46:48 +01:00
|
|
|
default:
|
|
|
|
return false;
|
2017-01-17 14:03:22 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);
|
|
|
|
|
2017-05-26 13:53:20 +02:00
|
|
|
/*
|
2018-05-30 09:26:07 +02:00
|
|
|
* Iterate list of requests and see if we can merge this bio with any
|
|
|
|
* of them.
|
2017-05-26 13:53:20 +02:00
|
|
|
*/
|
2018-05-30 09:26:07 +02:00
|
|
|
bool blk_mq_bio_list_merge(struct request_queue *q, struct list_head *list,
|
|
|
|
struct bio *bio)
|
2017-05-26 13:53:20 +02:00
|
|
|
{
|
|
|
|
struct request *rq;
|
|
|
|
int checked = 8;
|
|
|
|
|
2018-05-30 09:26:07 +02:00
|
|
|
list_for_each_entry_reverse(rq, list, queuelist) {
|
2017-05-26 13:53:20 +02:00
|
|
|
bool merged = false;
|
|
|
|
|
|
|
|
if (!checked--)
|
|
|
|
break;
|
|
|
|
|
|
|
|
if (!blk_rq_merge_ok(rq, bio))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
switch (blk_try_merge(rq, bio)) {
|
|
|
|
case ELEVATOR_BACK_MERGE:
|
|
|
|
if (blk_mq_sched_allow_merge(q, rq, bio))
|
|
|
|
merged = bio_attempt_back_merge(q, rq, bio);
|
|
|
|
break;
|
|
|
|
case ELEVATOR_FRONT_MERGE:
|
|
|
|
if (blk_mq_sched_allow_merge(q, rq, bio))
|
|
|
|
merged = bio_attempt_front_merge(q, rq, bio);
|
|
|
|
break;
|
|
|
|
case ELEVATOR_DISCARD_MERGE:
|
|
|
|
merged = bio_attempt_discard_merge(q, rq, bio);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
return merged;
|
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
2018-05-30 09:26:07 +02:00
|
|
|
EXPORT_SYMBOL_GPL(blk_mq_bio_list_merge);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Reverse check our software queue for entries that we could potentially
|
|
|
|
* merge with. Currently includes a hand-wavy stop count of 8, to not spend
|
|
|
|
* too much time checking for merges.
|
|
|
|
*/
|
|
|
|
static bool blk_mq_attempt_merge(struct request_queue *q,
|
|
|
|
struct blk_mq_ctx *ctx, struct bio *bio)
|
|
|
|
{
|
|
|
|
lockdep_assert_held(&ctx->lock);
|
|
|
|
|
|
|
|
if (blk_mq_bio_list_merge(q, &ctx->rq_list, bio)) {
|
|
|
|
ctx->rq_merged++;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
2017-05-26 13:53:20 +02:00
|
|
|
|
2017-01-17 14:03:22 +01:00
|
|
|
bool __blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio)
|
|
|
|
{
|
|
|
|
struct elevator_queue *e = q->elevator;
|
2017-05-26 13:53:20 +02:00
|
|
|
struct blk_mq_ctx *ctx = blk_mq_get_ctx(q);
|
|
|
|
struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);
|
|
|
|
bool ret = false;
|
2017-01-17 14:03:22 +01:00
|
|
|
|
2017-05-26 13:53:20 +02:00
|
|
|
if (e && e->type->ops.mq.bio_merge) {
|
2017-01-17 14:03:22 +01:00
|
|
|
blk_mq_put_ctx(ctx);
|
|
|
|
return e->type->ops.mq.bio_merge(hctx, bio);
|
|
|
|
}
|
|
|
|
|
2018-07-02 11:35:59 +02:00
|
|
|
if ((hctx->flags & BLK_MQ_F_SHOULD_MERGE) &&
|
|
|
|
!list_empty_careful(&ctx->rq_list)) {
|
2017-05-26 13:53:20 +02:00
|
|
|
/* default per sw-queue merge */
|
|
|
|
spin_lock(&ctx->lock);
|
|
|
|
ret = blk_mq_attempt_merge(q, ctx, bio);
|
|
|
|
spin_unlock(&ctx->lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
blk_mq_put_ctx(ctx);
|
|
|
|
return ret;
|
2017-01-17 14:03:22 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
bool blk_mq_sched_try_insert_merge(struct request_queue *q, struct request *rq)
|
|
|
|
{
|
|
|
|
return rq_mergeable(rq) && elv_attempt_insert_merge(q, rq);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(blk_mq_sched_try_insert_merge);
|
|
|
|
|
|
|
|
void blk_mq_sched_request_inserted(struct request *rq)
|
|
|
|
{
|
|
|
|
trace_block_rq_insert(rq->q, rq);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(blk_mq_sched_request_inserted);
|
|
|
|
|
2017-02-03 00:42:39 +01:00
|
|
|
static bool blk_mq_sched_bypass_insert(struct blk_mq_hw_ctx *hctx,
|
2017-11-02 16:24:36 +01:00
|
|
|
bool has_sched,
|
2017-02-03 00:42:39 +01:00
|
|
|
struct request *rq)
|
2017-01-17 14:03:22 +01:00
|
|
|
{
|
2017-11-02 16:24:36 +01:00
|
|
|
/* dispatch flush rq directly */
|
|
|
|
if (rq->rq_flags & RQF_FLUSH_SEQ) {
|
|
|
|
spin_lock(&hctx->lock);
|
|
|
|
list_add(&rq->queuelist, &hctx->dispatch);
|
|
|
|
spin_unlock(&hctx->lock);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2017-11-02 16:24:38 +01:00
|
|
|
if (has_sched)
|
2017-01-17 14:03:22 +01:00
|
|
|
rq->rq_flags |= RQF_SORTED;
|
|
|
|
|
2017-11-02 16:24:36 +01:00
|
|
|
return false;
|
2017-01-17 14:03:22 +01:00
|
|
|
}
|
|
|
|
|
2017-01-27 09:00:47 +01:00
|
|
|
void blk_mq_sched_insert_request(struct request *rq, bool at_head,
|
2018-01-17 17:25:58 +01:00
|
|
|
bool run_queue, bool async)
|
2017-01-27 09:00:47 +01:00
|
|
|
{
|
|
|
|
struct request_queue *q = rq->q;
|
|
|
|
struct elevator_queue *e = q->elevator;
|
|
|
|
struct blk_mq_ctx *ctx = rq->mq_ctx;
|
|
|
|
struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);
|
|
|
|
|
2017-11-02 16:24:36 +01:00
|
|
|
/* flush rq in flush machinery need to be dispatched directly */
|
|
|
|
if (!(rq->rq_flags & RQF_FLUSH_SEQ) && op_is_flush(rq->cmd_flags)) {
|
2017-11-02 16:24:38 +01:00
|
|
|
blk_insert_flush(rq);
|
|
|
|
goto run;
|
2017-01-27 09:00:47 +01:00
|
|
|
}
|
|
|
|
|
2017-11-02 16:24:38 +01:00
|
|
|
WARN_ON(e && (rq->tag != -1));
|
|
|
|
|
2017-11-02 16:24:36 +01:00
|
|
|
if (blk_mq_sched_bypass_insert(hctx, !!e, rq))
|
2017-02-03 00:42:39 +01:00
|
|
|
goto run;
|
|
|
|
|
2017-01-27 09:00:47 +01:00
|
|
|
if (e && e->type->ops.mq.insert_requests) {
|
|
|
|
LIST_HEAD(list);
|
|
|
|
|
|
|
|
list_add(&rq->queuelist, &list);
|
|
|
|
e->type->ops.mq.insert_requests(hctx, &list, at_head);
|
|
|
|
} else {
|
|
|
|
spin_lock(&ctx->lock);
|
|
|
|
__blk_mq_insert_request(hctx, rq, at_head);
|
|
|
|
spin_unlock(&ctx->lock);
|
|
|
|
}
|
|
|
|
|
2017-02-03 00:42:39 +01:00
|
|
|
run:
|
2017-01-27 09:00:47 +01:00
|
|
|
if (run_queue)
|
|
|
|
blk_mq_run_hw_queue(hctx, async);
|
|
|
|
}
|
|
|
|
|
|
|
|
void blk_mq_sched_insert_requests(struct request_queue *q,
|
|
|
|
struct blk_mq_ctx *ctx,
|
|
|
|
struct list_head *list, bool run_queue_async)
|
|
|
|
{
|
|
|
|
struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);
|
|
|
|
struct elevator_queue *e = hctx->queue->elevator;
|
|
|
|
|
|
|
|
if (e && e->type->ops.mq.insert_requests)
|
|
|
|
e->type->ops.mq.insert_requests(hctx, list, false);
|
2018-07-10 03:03:31 +02:00
|
|
|
else {
|
|
|
|
/*
|
|
|
|
* try to issue requests directly if the hw queue isn't
|
|
|
|
* busy in case of 'none' scheduler, and this way may save
|
|
|
|
* us one extra enqueue & dequeue to sw queue.
|
|
|
|
*/
|
|
|
|
if (!hctx->dispatch_busy && !e && !run_queue_async) {
|
|
|
|
blk_mq_try_issue_list_directly(hctx, list);
|
|
|
|
if (list_empty(list))
|
|
|
|
return;
|
|
|
|
}
|
2017-01-27 09:00:47 +01:00
|
|
|
blk_mq_insert_requests(hctx, ctx, list);
|
2018-07-10 03:03:31 +02:00
|
|
|
}
|
2017-01-27 09:00:47 +01:00
|
|
|
|
|
|
|
blk_mq_run_hw_queue(hctx, run_queue_async);
|
|
|
|
}
|
|
|
|
|
2017-01-17 14:03:22 +01:00
|
|
|
static void blk_mq_sched_free_tags(struct blk_mq_tag_set *set,
|
|
|
|
struct blk_mq_hw_ctx *hctx,
|
|
|
|
unsigned int hctx_idx)
|
|
|
|
{
|
|
|
|
if (hctx->sched_tags) {
|
|
|
|
blk_mq_free_rqs(set, hctx->sched_tags, hctx_idx);
|
|
|
|
blk_mq_free_rq_map(hctx->sched_tags);
|
|
|
|
hctx->sched_tags = NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-04-05 21:01:30 +02:00
|
|
|
static int blk_mq_sched_alloc_tags(struct request_queue *q,
|
|
|
|
struct blk_mq_hw_ctx *hctx,
|
|
|
|
unsigned int hctx_idx)
|
|
|
|
{
|
|
|
|
struct blk_mq_tag_set *set = q->tag_set;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
hctx->sched_tags = blk_mq_alloc_rq_map(set, hctx_idx, q->nr_requests,
|
|
|
|
set->reserved_tags);
|
|
|
|
if (!hctx->sched_tags)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
ret = blk_mq_alloc_rqs(set, hctx->sched_tags, hctx_idx, q->nr_requests);
|
|
|
|
if (ret)
|
|
|
|
blk_mq_sched_free_tags(set, hctx, hctx_idx);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2017-04-07 16:52:27 +02:00
|
|
|
static void blk_mq_sched_tags_teardown(struct request_queue *q)
|
2017-01-17 14:03:22 +01:00
|
|
|
{
|
|
|
|
struct blk_mq_tag_set *set = q->tag_set;
|
|
|
|
struct blk_mq_hw_ctx *hctx;
|
2017-04-05 21:01:30 +02:00
|
|
|
int i;
|
|
|
|
|
|
|
|
queue_for_each_hw_ctx(q, hctx, i)
|
|
|
|
blk_mq_sched_free_tags(set, hctx, i);
|
|
|
|
}
|
|
|
|
|
|
|
|
int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e)
|
|
|
|
{
|
|
|
|
struct blk_mq_hw_ctx *hctx;
|
2017-04-05 21:01:34 +02:00
|
|
|
struct elevator_queue *eq;
|
2017-04-05 21:01:30 +02:00
|
|
|
unsigned int i;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
if (!e) {
|
|
|
|
q->elevator = NULL;
|
2018-06-02 09:18:09 +02:00
|
|
|
q->nr_requests = q->tag_set->queue_depth;
|
2017-04-05 21:01:30 +02:00
|
|
|
return 0;
|
|
|
|
}
|
2017-01-17 14:03:22 +01:00
|
|
|
|
|
|
|
/*
|
2017-07-03 14:37:14 +02:00
|
|
|
* Default to double of smaller one between hw queue_depth and 128,
|
|
|
|
* since we don't split into sync/async like the old code did.
|
|
|
|
* Additionally, this is a per-hw queue depth.
|
2017-01-17 14:03:22 +01:00
|
|
|
*/
|
2017-07-03 14:37:14 +02:00
|
|
|
q->nr_requests = 2 * min_t(unsigned int, q->tag_set->queue_depth,
|
|
|
|
BLKDEV_MAX_RQ);
|
2017-01-17 14:03:22 +01:00
|
|
|
|
|
|
|
queue_for_each_hw_ctx(q, hctx, i) {
|
2017-04-05 21:01:30 +02:00
|
|
|
ret = blk_mq_sched_alloc_tags(q, hctx, i);
|
2017-01-17 14:03:22 +01:00
|
|
|
if (ret)
|
2017-04-05 21:01:30 +02:00
|
|
|
goto err;
|
2017-01-17 14:03:22 +01:00
|
|
|
}
|
|
|
|
|
2017-04-05 21:01:30 +02:00
|
|
|
ret = e->ops.mq.init_sched(q, e);
|
|
|
|
if (ret)
|
|
|
|
goto err;
|
2017-01-17 14:03:22 +01:00
|
|
|
|
2017-05-04 16:24:40 +02:00
|
|
|
blk_mq_debugfs_register_sched(q);
|
|
|
|
|
|
|
|
queue_for_each_hw_ctx(q, hctx, i) {
|
|
|
|
if (e->ops.mq.init_hctx) {
|
2017-04-05 21:01:34 +02:00
|
|
|
ret = e->ops.mq.init_hctx(hctx, i);
|
|
|
|
if (ret) {
|
|
|
|
eq = q->elevator;
|
|
|
|
blk_mq_exit_sched(q, eq);
|
|
|
|
kobject_put(&eq->kobj);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
}
|
2017-05-04 16:24:40 +02:00
|
|
|
blk_mq_debugfs_register_sched_hctx(q, hctx);
|
2017-04-05 21:01:34 +02:00
|
|
|
}
|
|
|
|
|
2017-01-17 14:03:22 +01:00
|
|
|
return 0;
|
|
|
|
|
2017-04-05 21:01:30 +02:00
|
|
|
err:
|
2017-04-07 16:52:27 +02:00
|
|
|
blk_mq_sched_tags_teardown(q);
|
|
|
|
q->elevator = NULL;
|
2017-04-05 21:01:30 +02:00
|
|
|
return ret;
|
2017-01-17 14:03:22 +01:00
|
|
|
}
|
2017-01-13 22:43:58 +01:00
|
|
|
|
2017-04-07 16:52:27 +02:00
|
|
|
void blk_mq_exit_sched(struct request_queue *q, struct elevator_queue *e)
|
|
|
|
{
|
2017-04-05 21:01:34 +02:00
|
|
|
struct blk_mq_hw_ctx *hctx;
|
|
|
|
unsigned int i;
|
|
|
|
|
2017-05-04 16:24:40 +02:00
|
|
|
queue_for_each_hw_ctx(q, hctx, i) {
|
|
|
|
blk_mq_debugfs_unregister_sched_hctx(hctx);
|
|
|
|
if (e->type->ops.mq.exit_hctx && hctx->sched_data) {
|
|
|
|
e->type->ops.mq.exit_hctx(hctx, i);
|
|
|
|
hctx->sched_data = NULL;
|
2017-04-05 21:01:34 +02:00
|
|
|
}
|
|
|
|
}
|
2017-05-04 16:24:40 +02:00
|
|
|
blk_mq_debugfs_unregister_sched(q);
|
2017-04-07 16:52:27 +02:00
|
|
|
if (e->type->ops.mq.exit_sched)
|
|
|
|
e->type->ops.mq.exit_sched(e);
|
|
|
|
blk_mq_sched_tags_teardown(q);
|
|
|
|
q->elevator = NULL;
|
|
|
|
}
|