diff options
| -rw-r--r-- | include/linux/io_uring_types.h | 3 | ||||
| -rw-r--r-- | io_uring/io_uring.c | 6 | ||||
| -rw-r--r-- | io_uring/msg_ring.c | 31 | ||||
| -rw-r--r-- | io_uring/msg_ring.h | 1 |
4 files changed, 39 insertions, 2 deletions
diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 1052a68fd68d..ede42dce1506 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -397,6 +397,9 @@ struct io_ring_ctx { struct callback_head poll_wq_task_work; struct list_head defer_list; + struct io_alloc_cache msg_cache; + spinlock_t msg_lock; + #ifdef CONFIG_NET_RX_BUSY_POLL struct list_head napi_list; /* track busy poll napi_id */ spinlock_t napi_lock; /* napi_list lock */ diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index cdeb94d2a26b..7ed1e009aaec 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -95,6 +95,7 @@ #include "futex.h" #include "napi.h" #include "uring_cmd.h" +#include "msg_ring.h" #include "memmap.h" #include "timeout.h" @@ -315,6 +316,9 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) sizeof(struct io_async_rw)); ret |= io_alloc_cache_init(&ctx->uring_cache, IO_ALLOC_CACHE_MAX, sizeof(struct uring_cache)); + spin_lock_init(&ctx->msg_lock); + ret |= io_alloc_cache_init(&ctx->msg_cache, IO_ALLOC_CACHE_MAX, + sizeof(struct io_kiocb)); ret |= io_futex_cache_init(ctx); if (ret) goto err; @@ -351,6 +355,7 @@ err: io_alloc_cache_free(&ctx->netmsg_cache, io_netmsg_cache_free); io_alloc_cache_free(&ctx->rw_cache, io_rw_cache_free); io_alloc_cache_free(&ctx->uring_cache, kfree); + io_alloc_cache_free(&ctx->msg_cache, io_msg_cache_free); io_futex_cache_free(ctx); kfree(ctx->cancel_table.hbs); kfree(ctx->cancel_table_locked.hbs); @@ -2599,6 +2604,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) io_alloc_cache_free(&ctx->netmsg_cache, io_netmsg_cache_free); io_alloc_cache_free(&ctx->rw_cache, io_rw_cache_free); io_alloc_cache_free(&ctx->uring_cache, kfree); + io_alloc_cache_free(&ctx->msg_cache, io_msg_cache_free); io_futex_cache_free(ctx); io_destroy_buffers(ctx); mutex_unlock(&ctx->uring_lock); diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c index ad7d67d44461..47a754e83b49 100644 --- a/io_uring/msg_ring.c +++ b/io_uring/msg_ring.c @@ -11,6 +11,7 @@ #include "io_uring.h" #include "rsrc.h" #include "filetable.h" +#include "alloc_cache.h" #include "msg_ring.h" /* All valid masks for MSG_RING */ @@ -75,7 +76,13 @@ static void io_msg_tw_complete(struct io_kiocb *req, struct io_tw_state *ts) struct io_ring_ctx *ctx = req->ctx; io_add_aux_cqe(ctx, req->cqe.user_data, req->cqe.res, req->cqe.flags); - kmem_cache_free(req_cachep, req); + if (spin_trylock(&ctx->msg_lock)) { + if (io_alloc_cache_put(&ctx->msg_cache, req)) + req = NULL; + spin_unlock(&ctx->msg_lock); + } + if (req) + kfree(req); percpu_ref_put(&ctx->refs); } @@ -91,6 +98,19 @@ static void io_msg_remote_post(struct io_ring_ctx *ctx, struct io_kiocb *req, io_req_task_work_add_remote(req, ctx, IOU_F_TWQ_LAZY_WAKE); } +static struct io_kiocb *io_msg_get_kiocb(struct io_ring_ctx *ctx) +{ + struct io_kiocb *req = NULL; + + if (spin_trylock(&ctx->msg_lock)) { + req = io_alloc_cache_get(&ctx->msg_cache); + spin_unlock(&ctx->msg_lock); + } + if (req) + return req; + return kmem_cache_alloc(req_cachep, GFP_KERNEL | __GFP_NOWARN); +} + static int io_msg_data_remote(struct io_kiocb *req) { struct io_ring_ctx *target_ctx = req->file->private_data; @@ -98,7 +118,7 @@ static int io_msg_data_remote(struct io_kiocb *req) struct io_kiocb *target; u32 flags = 0; - target = kmem_cache_alloc(req_cachep, GFP_KERNEL); + target = io_msg_get_kiocb(req->ctx); if (unlikely(!target)) return -ENOMEM; @@ -296,3 +316,10 @@ done: io_req_set_res(req, ret, 0); return IOU_OK; } + +void io_msg_cache_free(const void *entry) +{ + struct io_kiocb *req = (struct io_kiocb *) entry; + + kmem_cache_free(req_cachep, req); +} diff --git a/io_uring/msg_ring.h b/io_uring/msg_ring.h index 3987ee6c0e5f..3030f3942f0f 100644 --- a/io_uring/msg_ring.h +++ b/io_uring/msg_ring.h @@ -3,3 +3,4 @@ int io_msg_ring_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); int io_msg_ring(struct io_kiocb *req, unsigned int issue_flags); void io_msg_ring_cleanup(struct io_kiocb *req); +void io_msg_cache_free(const void *entry); |