diff options
| author | Pavel Begunkov <asml.silence@gmail.com> | 2023-08-25 00:53:36 +0200 |
|---|---|---|
| committer | Jens Axboe <axboe@kernel.dk> | 2023-08-25 01:16:20 +0200 |
| commit | 0aa7aa5f766933d4f91b22d9658cd688e1f15dab (patch) | |
| tree | 099d4801e6657eb0f0663ffb149c73fb9e97ca9e /io_uring | |
| parent | io_uring: separate task_work/waiting cache line (diff) | |
| download | linux-0aa7aa5f766933d4f91b22d9658cd688e1f15dab.tar.xz linux-0aa7aa5f766933d4f91b22d9658cd688e1f15dab.zip | |
io_uring: move multishot cqe cache in ctx
We cache multishot CQEs before flushing them to the CQ in
submit_state.cqe. It's a 16 entry cache totalling 256 bytes in the
middle of the io_submit_state structure. Move it out of there, it
should help with CPU caches for the submission state, and shouldn't
affect cached CQEs.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/dbe1f39c043ee23da918836be44fcec252ce6711.1692916914.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Diffstat (limited to 'io_uring')
| -rw-r--r-- | io_uring/io_uring.c | 6 |
1 files changed, 3 insertions, 3 deletions
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index a6eea3938802..88599852af82 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -880,7 +880,7 @@ static void __io_flush_post_cqes(struct io_ring_ctx *ctx) lockdep_assert_held(&ctx->uring_lock); for (i = 0; i < state->cqes_count; i++) { - struct io_uring_cqe *cqe = &state->cqes[i]; + struct io_uring_cqe *cqe = &ctx->completion_cqes[i]; if (!io_fill_cqe_aux(ctx, cqe->user_data, cqe->res, cqe->flags)) { if (ctx->task_complete) { @@ -931,7 +931,7 @@ bool io_fill_cqe_req_aux(struct io_kiocb *req, bool defer, s32 res, u32 cflags) lockdep_assert_held(&ctx->uring_lock); - if (ctx->submit_state.cqes_count == ARRAY_SIZE(ctx->submit_state.cqes)) { + if (ctx->submit_state.cqes_count == ARRAY_SIZE(ctx->completion_cqes)) { __io_cq_lock(ctx); __io_flush_post_cqes(ctx); /* no need to flush - flush is deferred */ @@ -945,7 +945,7 @@ bool io_fill_cqe_req_aux(struct io_kiocb *req, bool defer, s32 res, u32 cflags) if (test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq)) return false; - cqe = &ctx->submit_state.cqes[ctx->submit_state.cqes_count++]; + cqe = &ctx->completion_cqes[ctx->submit_state.cqes_count++]; cqe->user_data = user_data; cqe->res = res; cqe->flags = cflags; |