From 5b13bc8a3fd519d86e5b1a0b1d1b996cace62f3f Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 24 Nov 2021 07:28:56 +0100 Subject: blk-mq: cleanup request allocation Refactor the request alloction so that blk_mq_get_cached_request tries to find a cached request first, and the entirely separate and now self contained blk_mq_get_new_requests allocates one or more requests if that is not possible. There is a small change in behavior as submit_bio_checks is called twice now if a cached request is present but can't be used, but that is a small price to pay for unwinding this code. Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20211124062856.1444266-1-hch@lst.de Signed-off-by: Jens Axboe --- block/blk-mq.c | 90 +++++++++++++++++++++++++--------------------------------- 1 file changed, 38 insertions(+), 52 deletions(-) (limited to 'block/blk-mq.c') diff --git a/block/blk-mq.c b/block/blk-mq.c index 4bdc3bc54ea3..a89a624dd1df 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -2717,8 +2717,12 @@ static struct request *blk_mq_get_new_requests(struct request_queue *q, }; struct request *rq; - if (blk_mq_attempt_bio_merge(q, bio, nsegs)) + if (unlikely(bio_queue_enter(bio))) return NULL; + if (unlikely(!submit_bio_checks(bio))) + goto queue_exit; + if (blk_mq_attempt_bio_merge(q, bio, nsegs)) + goto queue_exit; rq_qos_throttle(q, bio); @@ -2729,64 +2733,44 @@ static struct request *blk_mq_get_new_requests(struct request_queue *q, } rq = __blk_mq_alloc_requests(&data); - if (rq) - return rq; + if (!rq) + goto fail; + return rq; +fail: rq_qos_cleanup(q, bio); if (bio->bi_opf & REQ_NOWAIT) bio_wouldblock_error(bio); - +queue_exit: + blk_queue_exit(q); return NULL; } -static inline bool blk_mq_can_use_cached_rq(struct request *rq, struct bio *bio) -{ - if (blk_mq_get_hctx_type(bio->bi_opf) != rq->mq_hctx->type) - return false; - - if (op_is_flush(rq->cmd_flags) != op_is_flush(bio->bi_opf)) - return false; - - return true; -} - -static inline struct request *blk_mq_get_request(struct request_queue *q, - struct blk_plug *plug, - struct bio *bio, - unsigned int nsegs) +static inline struct request *blk_mq_get_cached_request(struct request_queue *q, + struct blk_plug *plug, struct bio *bio, unsigned int nsegs) { struct request *rq; - bool checked = false; - if (plug) { - rq = rq_list_peek(&plug->cached_rq); - if (rq && rq->q == q) { - if (unlikely(!submit_bio_checks(bio))) - return NULL; - if (blk_mq_attempt_bio_merge(q, bio, nsegs)) - return NULL; - checked = true; - if (!blk_mq_can_use_cached_rq(rq, bio)) - goto fallback; - rq->cmd_flags = bio->bi_opf; - plug->cached_rq = rq_list_next(rq); - INIT_LIST_HEAD(&rq->queuelist); - rq_qos_throttle(q, bio); - return rq; - } - } + if (!plug) + return NULL; + rq = rq_list_peek(&plug->cached_rq); + if (!rq || rq->q != q) + return NULL; -fallback: - if (unlikely(bio_queue_enter(bio))) + if (unlikely(!submit_bio_checks(bio))) return NULL; - if (unlikely(!checked && !submit_bio_checks(bio))) - goto out_put; - rq = blk_mq_get_new_requests(q, plug, bio, nsegs); - if (rq) - return rq; -out_put: - blk_queue_exit(q); - return NULL; + if (blk_mq_attempt_bio_merge(q, bio, nsegs)) + return NULL; + if (blk_mq_get_hctx_type(bio->bi_opf) != rq->mq_hctx->type) + return NULL; + if (op_is_flush(rq->cmd_flags) != op_is_flush(bio->bi_opf)) + return NULL; + + rq->cmd_flags = bio->bi_opf; + plug->cached_rq = rq_list_next(rq); + INIT_LIST_HEAD(&rq->queuelist); + rq_qos_throttle(q, bio); + return rq; } /** @@ -2805,9 +2789,9 @@ out_put: void blk_mq_submit_bio(struct bio *bio) { struct request_queue *q = bdev_get_queue(bio->bi_bdev); + struct blk_plug *plug = blk_mq_plug(q, bio); const int is_sync = op_is_sync(bio->bi_opf); struct request *rq; - struct blk_plug *plug; unsigned int nr_segs = 1; blk_status_t ret; @@ -2821,10 +2805,12 @@ void blk_mq_submit_bio(struct bio *bio) if (!bio_integrity_prep(bio)) return; - plug = blk_mq_plug(q, bio); - rq = blk_mq_get_request(q, plug, bio, nr_segs); - if (unlikely(!rq)) - return; + rq = blk_mq_get_cached_request(q, plug, bio, nr_segs); + if (!rq) { + rq = blk_mq_get_new_requests(q, plug, bio, nr_segs); + if (unlikely(!rq)) + return; + } trace_block_getrq(bio); -- cgit v1.2.3