@@ -4793,29 +4793,37 @@ static int bfq_dispatch_requests(struct blk_mq_hw_ctx *hctx,
struct list_head *list)
{
struct bfq_data *bfqd = hctx->queue->elevator->elevator_data;
+ unsigned int batch_reqs = queue_max_batch_requests(hctx->queue) ? : 1;
struct request *rq;
struct bfq_queue *in_serv_queue;
bool waiting_rq, idle_timer_disabled;
+ int i;
- spin_lock_irq(&bfqd->lock);
+ for (i = 0; i < batch_reqs; i++) {
+ spin_lock_irq(&bfqd->lock);
- in_serv_queue = bfqd->in_service_queue;
- waiting_rq = in_serv_queue && bfq_bfqq_wait_request(in_serv_queue);
+ in_serv_queue = bfqd->in_service_queue;
+ waiting_rq = in_serv_queue && bfq_bfqq_wait_request(in_serv_queue);
- rq = __bfq_dispatch_request(hctx);
+ rq = __bfq_dispatch_request(hctx);
- idle_timer_disabled =
- waiting_rq && !bfq_bfqq_wait_request(in_serv_queue);
+ idle_timer_disabled =
+ waiting_rq && !bfq_bfqq_wait_request(in_serv_queue);
- spin_unlock_irq(&bfqd->lock);
+ spin_unlock_irq(&bfqd->lock);
- bfq_update_dispatch_stats(hctx->queue, rq, in_serv_queue,
- idle_timer_disabled);
+ bfq_update_dispatch_stats(hctx->queue, rq, in_serv_queue,
+ idle_timer_disabled);
- if (!rq)
- return 0;
+ if (!rq) {
+ if (list_empty(list))
+ return 0;
- list_add(&rq->queuelist, list);
+ return 1;
+ }
+
+ list_add(&rq->queuelist, list);
+ }
return 1;
}
@@ -1193,8 +1193,6 @@ bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list,
if (list_empty(list))
return false;
- WARN_ON(!list_is_singular(list) && got_budget);
-
/*
* Now process all the entries, sending them to the driver.
*/
@@ -59,6 +59,7 @@ void blk_set_default_limits(struct queue_limits *lim)
lim->io_opt = 0;
lim->misaligned = 0;
lim->zoned = BLK_ZONED_NONE;
+ lim->max_batch_reqs = 1;
}
EXPORT_SYMBOL(blk_set_default_limits);
@@ -871,6 +872,18 @@ bool blk_queue_can_use_dma_map_merging(struct request_queue *q,
}
EXPORT_SYMBOL_GPL(blk_queue_can_use_dma_map_merging);
+/**
+ * blk_queue_max_batch_requests - set max requests for batch processing
+ * @q: the request queue for the device
+ * @max_batch_requests: maximum number of requests in a batch
+ **/
+void blk_queue_max_batch_requests(struct request_queue *q,
+ unsigned int max_batch_requests)
+{
+ q->limits.max_batch_reqs = max_batch_requests;
+}
+EXPORT_SYMBOL(blk_queue_max_batch_requests);
+
static int __init blk_settings_init(void)
{
blk_max_low_pfn = max_low_pfn - 1;
@@ -801,50 +801,56 @@ static int kyber_dispatch_requests(struct blk_mq_hw_ctx *hctx,
{
struct kyber_queue_data *kqd = hctx->queue->elevator->elevator_data;
struct kyber_hctx_data *khd = hctx->sched_data;
+ unsigned int batch_reqs = queue_max_batch_requests(hctx->queue) ? : 1;
struct request *rq;
- int i, ret = 0;
+ int i, j, ret = 0;
spin_lock(&khd->lock);
- /*
- * First, if we are still entitled to batch, try to dispatch a request
- * from the batch.
- */
- if (khd->batching < kyber_batch_size[khd->cur_domain]) {
- rq = kyber_dispatch_cur_domain(kqd, khd, hctx);
- if (rq) {
- list_add(&rq->queuelist, list);
- ret = 1;
- goto out;
+ for (j = 0; j < batch_reqs; j++) {
+ /*
+ * First, if we are still entitled to batch, try to dispatch a
+ * request from the batch.
+ */
+ if (khd->batching < kyber_batch_size[khd->cur_domain]) {
+ rq = kyber_dispatch_cur_domain(kqd, khd, hctx);
+ if (rq) {
+ list_add(&rq->queuelist, list);
+ ret = 1;
+ continue;
+ }
}
- }
-
- /*
- * Either,
- * 1. We were no longer entitled to a batch.
- * 2. The domain we were batching didn't have any requests.
- * 3. The domain we were batching was out of tokens.
- *
- * Start another batch. Note that this wraps back around to the original
- * domain if no other domains have requests or tokens.
- */
- khd->batching = 0;
- for (i = 0; i < KYBER_NUM_DOMAINS; i++) {
- if (khd->cur_domain == KYBER_NUM_DOMAINS - 1)
- khd->cur_domain = 0;
- else
- khd->cur_domain++;
- rq = kyber_dispatch_cur_domain(kqd, khd, hctx);
- if (rq) {
- list_add(&rq->queuelist, list);
- ret = 1;
- goto out;
+ /*
+ * Either,
+ * 1. We were no longer entitled to a batch.
+ * 2. The domain we were batching didn't have any requests.
+ * 3. The domain we were batching was out of tokens.
+ *
+ * Start another batch. Note that this wraps back around to the
+ * original domain if no other domains have requests or tokens.
+ */
+ khd->batching = 0;
+ for (i = 0; i < KYBER_NUM_DOMAINS; i++) {
+ if (khd->cur_domain == KYBER_NUM_DOMAINS - 1)
+ khd->cur_domain = 0;
+ else
+ khd->cur_domain++;
+
+ rq = kyber_dispatch_cur_domain(kqd, khd, hctx);
+ if (rq) {
+ list_add(&rq->queuelist, list);
+ ret = 1;
+ break;
+ }
}
}
-out:
spin_unlock(&khd->lock);
+
+ if (list_empty(list))
+ ret = 0;
+
return ret;
}
@@ -382,16 +382,24 @@ static int dd_dispatch_requests(struct blk_mq_hw_ctx *hctx,
struct list_head *list)
{
struct deadline_data *dd = hctx->queue->elevator->elevator_data;
+ unsigned int batch_reqs = queue_max_batch_requests(hctx->queue) ? : 1;
struct request *rq;
+ int i;
- spin_lock(&dd->lock);
- rq = __dd_dispatch_request(dd);
- spin_unlock(&dd->lock);
+ for (i = 0; i < batch_reqs; i++) {
+ spin_lock(&dd->lock);
+ rq = __dd_dispatch_request(dd);
+ spin_unlock(&dd->lock);
- if (!rq)
- return 0;
+ if (!rq) {
+ if (list_empty(list))
+ return 0;
- list_add(&rq->queuelist, list);
+ return 1;
+ }
+
+ list_add(&rq->queuelist, list);
+ }
return 1;
}
@@ -338,6 +338,7 @@ struct queue_limits {
unsigned int max_write_zeroes_sectors;
unsigned int discard_granularity;
unsigned int discard_alignment;
+ unsigned int max_batch_reqs;
unsigned short max_segments;
unsigned short max_integrity_segments;
@@ -1109,6 +1110,8 @@ extern void blk_queue_required_elevator_features(struct request_queue *q,
unsigned int features);
extern bool blk_queue_can_use_dma_map_merging(struct request_queue *q,
struct device *dev);
+extern void blk_queue_max_batch_requests(struct request_queue *q,
+ unsigned int max_batch_requests);
/*
* Number of physical segments as sent to the device.
@@ -1291,6 +1294,11 @@ static inline unsigned int queue_max_segment_size(const struct request_queue *q)
return q->limits.max_segment_size;
}
+static inline unsigned int queue_max_batch_requests(const struct request_queue *q)
+{
+ return q->limits.max_batch_reqs;
+}
+
static inline unsigned queue_logical_block_size(const struct request_queue *q)
{
int retval = 512;
As we know, some SD/MMC host controllers can support packed request, that means we can package several requests to host controller at one time to improve performence. So the hardware driver expects the blk-mq can dispatch a batch of requests at one time, and driver can use bd.last to indicate if it is the last request in the batch to help to combine requests as much as possible. Thus we should add batch requests setting from the block driver to tell the scheduler how many requests can be dispatched in a batch, as well as changing the scheduler to dispatch more than one request if setting the maximum batch requests number. Signed-off-by: Baolin Wang <baolin.wang7@gmail.com> --- block/bfq-iosched.c | 32 ++++++++++++++-------- block/blk-mq.c | 2 -- block/blk-settings.c | 13 +++++++++ block/kyber-iosched.c | 74 +++++++++++++++++++++++++++----------------------- block/mq-deadline.c | 20 ++++++++++---- include/linux/blkdev.h | 8 ++++++ 6 files changed, 95 insertions(+), 54 deletions(-)