diff options
| author | Pavel Begunkov <asml.silence@gmail.com> | 2022-11-04 11:59:45 +0100 |
|---|---|---|
| committer | Jens Axboe <axboe@kernel.dk> | 2022-11-21 15:38:31 +0100 |
| commit | 40725d1b960f19a11a1ebd1ab537844ebf39347c (patch) | |
| tree | 764d4e8ba38c563b9a2d7f7dde7df42b26792079 /io_uring/notif.c | |
| parent | io_uring/net: inline io_notif_flush() (diff) | |
| download | linux-40725d1b960f19a11a1ebd1ab537844ebf39347c.tar.xz linux-40725d1b960f19a11a1ebd1ab537844ebf39347c.zip | |
io_uring: move zc reporting from the hot path
Add custom tw and notif callbacks on top of usual bits also handling zc
reporting. That moves it from the hot path.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/40de4a6409042478e1f35adc4912e23226cb1b5c.1667557923.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Diffstat (limited to 'io_uring/notif.c')
| -rw-r--r-- | io_uring/notif.c | 31 |
1 files changed, 27 insertions, 4 deletions
diff --git a/io_uring/notif.c b/io_uring/notif.c index c287adf24e66..9864bde3e2ef 100644 --- a/io_uring/notif.c +++ b/io_uring/notif.c @@ -18,11 +18,17 @@ static void __io_notif_complete_tw(struct io_kiocb *notif, bool *locked) __io_unaccount_mem(ctx->user, nd->account_pages); nd->account_pages = 0; } + io_req_task_complete(notif, locked); +} + +static void io_notif_complete_tw_ext(struct io_kiocb *notif, bool *locked) +{ + struct io_notif_data *nd = io_notif_to_data(notif); if (nd->zc_report && (nd->zc_copied || !nd->zc_used)) notif->cqe.res |= IORING_NOTIF_USAGE_ZC_COPIED; - io_req_task_complete(notif, locked); + __io_notif_complete_tw(notif, locked); } static void io_tx_ubuf_callback(struct sk_buff *skb, struct ubuf_info *uarg, @@ -31,15 +37,33 @@ static void io_tx_ubuf_callback(struct sk_buff *skb, struct ubuf_info *uarg, struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg); struct io_kiocb *notif = cmd_to_io_kiocb(nd); + if (refcount_dec_and_test(&uarg->refcnt)) + io_req_task_work_add(notif); +} + +static void io_tx_ubuf_callback_ext(struct sk_buff *skb, struct ubuf_info *uarg, + bool success) +{ + struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg); + if (nd->zc_report) { if (success && !nd->zc_used && skb) WRITE_ONCE(nd->zc_used, true); else if (!success && !nd->zc_copied) WRITE_ONCE(nd->zc_copied, true); } + io_tx_ubuf_callback(skb, uarg, success); +} - if (refcount_dec_and_test(&uarg->refcnt)) - io_req_task_work_add(notif); +void io_notif_set_extended(struct io_kiocb *notif) +{ + struct io_notif_data *nd = io_notif_to_data(notif); + + nd->zc_report = false; + nd->zc_used = false; + nd->zc_copied = false; + notif->io_task_work.func = io_notif_complete_tw_ext; + io_notif_to_data(notif)->uarg.callback = io_tx_ubuf_callback_ext; } struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx) @@ -63,7 +87,6 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx) nd->account_pages = 0; nd->uarg.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN; nd->uarg.callback = io_tx_ubuf_callback; - nd->zc_report = nd->zc_used = nd->zc_copied = false; refcount_set(&nd->uarg.refcnt, 1); return notif; } |