diff options
| author | Pavel Begunkov <asml.silence@gmail.com> | 2023-04-04 14:39:49 +0200 |
|---|---|---|
| committer | Jens Axboe <axboe@kernel.dk> | 2023-04-04 17:30:39 +0200 |
| commit | ef8ae64ffa9578c12e44de42604004c2cc3e9c27 (patch) | |
| tree | a9aeabbb02273d7ca7ce7c38eebd96665b100156 /io_uring/io_uring.c | |
| parent | io_uring: io_free_req() via tw (diff) | |
| download | linux-ef8ae64ffa9578c12e44de42604004c2cc3e9c27.tar.xz linux-ef8ae64ffa9578c12e44de42604004c2cc3e9c27.zip | |
io_uring/rsrc: protect node refs with uring_lock
Currently, for nodes we have an atomic counter and some cached
(non-atomic) refs protected by uring_lock. Let's put all ref
manipulations under uring_lock and get rid of the atomic part.
It's free as in all cases we care about we already hold the lock.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/25b142feed7d831008257d90c8b17c0115d4fc15.1680576071.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Diffstat (limited to 'io_uring/io_uring.c')
| -rw-r--r-- | io_uring/io_uring.c | 18 |
1 files changed, 12 insertions, 6 deletions
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 98320f4b0bca..36a76c7b34f0 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -967,7 +967,7 @@ bool io_aux_cqe(struct io_ring_ctx *ctx, bool defer, u64 user_data, s32 res, u32 return true; } -static void __io_req_complete_post(struct io_kiocb *req) +static void __io_req_complete_post(struct io_kiocb *req, unsigned issue_flags) { struct io_ring_ctx *ctx = req->ctx; struct io_rsrc_node *rsrc_node = NULL; @@ -1003,7 +1003,11 @@ static void __io_req_complete_post(struct io_kiocb *req) } io_cq_unlock_post(ctx); - io_put_rsrc_node(rsrc_node); + if (rsrc_node) { + io_ring_submit_lock(ctx, issue_flags); + io_put_rsrc_node(rsrc_node); + io_ring_submit_unlock(ctx, issue_flags); + } } void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags) @@ -1013,12 +1017,12 @@ void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags) io_req_task_work_add(req); } else if (!(issue_flags & IO_URING_F_UNLOCKED) || !(req->ctx->flags & IORING_SETUP_IOPOLL)) { - __io_req_complete_post(req); + __io_req_complete_post(req, issue_flags); } else { struct io_ring_ctx *ctx = req->ctx; mutex_lock(&ctx->uring_lock); - __io_req_complete_post(req); + __io_req_complete_post(req, issue_flags & ~IO_URING_F_UNLOCKED); mutex_unlock(&ctx->uring_lock); } } @@ -1120,7 +1124,10 @@ static __cold void io_free_req_tw(struct io_kiocb *req, struct io_tw_state *ts) { struct io_ring_ctx *ctx = req->ctx; - io_put_rsrc_node(req->rsrc_node); + if (req->rsrc_node) { + io_tw_lock(ctx, ts); + io_put_rsrc_node(req->rsrc_node); + } io_dismantle_req(req); io_put_task_remote(req->task, 1); @@ -2790,7 +2797,6 @@ static void io_req_caches_free(struct io_ring_ctx *ctx) static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) { io_sq_thread_finish(ctx); - io_rsrc_refs_drop(ctx); /* __io_rsrc_put_work() may need uring_lock to progress, wait w/o it */ io_wait_rsrc_data(ctx->buf_data); io_wait_rsrc_data(ctx->file_data); |