diff options
author | Jens Axboe <axboe@kernel.dk> | 2024-01-11 21:34:33 +0100 |
---|---|---|
committer | Jens Axboe <axboe@kernel.dk> | 2024-01-11 21:37:31 +0100 |
commit | 3f302388d45855c0b24802e7b414e3fb29f172e3 (patch) | |
tree | 073876c607431fdab4f0ff5f267c821bf96a0fcc /io_uring | |
parent | io_uring/rw: cleanup io_rw_done() (diff) | |
download | linux-3f302388d45855c0b24802e7b414e3fb29f172e3.tar.xz linux-3f302388d45855c0b24802e7b414e3fb29f172e3.zip |
io_uring/rsrc: improve code generation for fixed file assignment
For the normal read/write path, we have already locked the ring
submission side when assigning the file. This causes branch
mispredictions when we then check and try and lock again in
io_req_set_rsrc_node(). As this is a very hot path, this matters.
Add a basic helper that already assumes we already have it locked,
and use that in io_file_get_fixed().
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Diffstat (limited to 'io_uring')
-rw-r--r-- | io_uring/io_uring.c | 5 | ||||
-rw-r--r-- | io_uring/rsrc.h | 14 |
2 files changed, 12 insertions, 7 deletions
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 4afb911fc042..50c9f04bc193 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2000,9 +2000,10 @@ inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd, goto out; fd = array_index_nospec(fd, ctx->nr_user_files); slot = io_fixed_file_slot(&ctx->file_table, fd); - file = io_slot_file(slot); + if (!req->rsrc_node) + __io_req_set_rsrc_node(req, ctx); req->flags |= io_slot_flags(slot); - io_req_set_rsrc_node(req, ctx, 0); + file = io_slot_file(slot); out: io_ring_submit_unlock(ctx, issue_flags); return file; diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h index 7238b9cfe33b..c6f199bbee28 100644 --- a/io_uring/rsrc.h +++ b/io_uring/rsrc.h @@ -102,17 +102,21 @@ static inline void io_charge_rsrc_node(struct io_ring_ctx *ctx, node->refs++; } +static inline void __io_req_set_rsrc_node(struct io_kiocb *req, + struct io_ring_ctx *ctx) +{ + lockdep_assert_held(&ctx->uring_lock); + req->rsrc_node = ctx->rsrc_node; + io_charge_rsrc_node(ctx, ctx->rsrc_node); +} + static inline void io_req_set_rsrc_node(struct io_kiocb *req, struct io_ring_ctx *ctx, unsigned int issue_flags) { if (!req->rsrc_node) { io_ring_submit_lock(ctx, issue_flags); - - lockdep_assert_held(&ctx->uring_lock); - - req->rsrc_node = ctx->rsrc_node; - io_charge_rsrc_node(ctx, ctx->rsrc_node); + __io_req_set_rsrc_node(req, ctx); io_ring_submit_unlock(ctx, issue_flags); } } |