diff options
author | Jens Axboe <axboe@kernel.dk> | 2021-12-01 23:01:51 +0100 |
---|---|---|
committer | Jens Axboe <axboe@kernel.dk> | 2021-12-16 16:42:06 +0100 |
commit | 5581a5ddfe8d0ede1749bae1f7662fa739f83813 (patch) | |
tree | 95afb2723a75b3b2cf46b21993ac026ce5cf0c82 /block/blk-mq.c | |
parent | block: make queue stat accounting a reference (diff) | |
download | linux-5581a5ddfe8d0ede1749bae1f7662fa739f83813.tar.xz linux-5581a5ddfe8d0ede1749bae1f7662fa739f83813.zip |
block: add completion handler for fast path
The batched completions only deal with non-partial requests anyway,
and it doesn't deal with any requests that have errors. Add a completion
handler that assumes it's a full request and that it's all being ended
successfully.
Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Diffstat (limited to 'block/blk-mq.c')
-rw-r--r-- | block/blk-mq.c | 43 |
1 files changed, 42 insertions, 1 deletions
diff --git a/block/blk-mq.c b/block/blk-mq.c index 103c0f58853c..75154cc788db 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -709,6 +709,47 @@ static void blk_print_req_error(struct request *req, blk_status_t status) IOPRIO_PRIO_CLASS(req->ioprio)); } +/* + * Fully end IO on a request. Does not support partial completions, or + * errors. + */ +static void blk_complete_request(struct request *req) +{ + const bool is_flush = (req->rq_flags & RQF_FLUSH_SEQ) != 0; + int total_bytes = blk_rq_bytes(req); + struct bio *bio = req->bio; + + trace_block_rq_complete(req, BLK_STS_OK, total_bytes); + + if (!bio) + return; + +#ifdef CONFIG_BLK_DEV_INTEGRITY + if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ) + req->q->integrity.profile->complete_fn(req, total_bytes); +#endif + + blk_account_io_completion(req, total_bytes); + + do { + struct bio *next = bio->bi_next; + + /* Completion has already been traced */ + bio_clear_flag(bio, BIO_TRACE_COMPLETION); + if (!is_flush) + bio_endio(bio); + bio = next; + } while (bio); + + /* + * Reset counters so that the request stacking driver + * can find how many bytes remain in the request + * later. + */ + req->bio = NULL; + req->__data_len = 0; +} + /** * blk_update_request - Complete multiple bytes without completing the request * @req: the request being processed @@ -922,7 +963,7 @@ void blk_mq_end_request_batch(struct io_comp_batch *iob) prefetch(rq->bio); prefetch(rq->rq_next); - blk_update_request(rq, BLK_STS_OK, blk_rq_bytes(rq)); + blk_complete_request(rq); if (iob->need_ts) __blk_mq_end_request_acct(rq, now); |