From e4a0d3e720e7e508749c1439b5ba3aff56c92976 Mon Sep 17 00:00:00 2001 From: Pavel Emelyanov Date: Thu, 18 Sep 2014 19:56:17 +0400 Subject: aio: Make it possible to remap aio ring There are actually two issues this patch addresses. Let me start with the one I tried to solve in the beginning. So, in the checkpoint-restore project (criu) we try to dump tasks' state and restore one back exactly as it was. One of the tasks' state bits is rings set up with io_setup() call. There's (almost) no problems in dumping them, there's a problem restoring them -- if I dump a task with aio ring originally mapped at address A, I want to restore one back at exactly the same address A. Unfortunately, the io_setup() does not allow for that -- it mmaps the ring at whatever place mm finds appropriate (it calls do_mmap_pgoff() with zero address and without the MAP_FIXED flag). To make restore possible I'm going to mremap() the freshly created ring into the address A (under which it was seen before dump). The problem is that the ring's virtual address is passed back to the user-space as the context ID and this ID is then used as search key by all the other io_foo() calls. Reworking this ID to be just some integer doesn't seem to work, as this value is already used by libaio as a pointer using which this library accesses memory for aio meta-data. So, to make restore work we need to make sure that a) ring is mapped at desired virtual address b) kioctx->user_id matches this value Having said that, the patch makes mremap() on aio region update the kioctx's user_id and mmap_base values. Here appears the 2nd issue I mentioned in the beginning of this mail. If (regardless of the C/R dances I do) someone creates an io context with io_setup(), then mremap()-s the ring and then destroys the context, the kill_ioctx() routine will call munmap() on wrong (old) address. This will result in a) aio ring remaining in memory and b) some other vma get unexpectedly unmapped. What do you think? Signed-off-by: Pavel Emelyanov Acked-by: Dmitry Monakhov Signed-off-by: Benjamin LaHaise --- fs/aio.c | 25 +++++++++++++++++++++++++ include/linux/fs.h | 1 + mm/mremap.c | 3 ++- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/fs/aio.c b/fs/aio.c index 14b93159ef83..bfab55607a4d 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -286,12 +286,37 @@ static void aio_free_ring(struct kioctx *ctx) static int aio_ring_mmap(struct file *file, struct vm_area_struct *vma) { + vma->vm_flags |= VM_DONTEXPAND; vma->vm_ops = &generic_file_vm_ops; return 0; } +static void aio_ring_remap(struct file *file, struct vm_area_struct *vma) +{ + struct mm_struct *mm = vma->vm_mm; + struct kioctx_table *table; + int i; + + spin_lock(&mm->ioctx_lock); + rcu_read_lock(); + table = rcu_dereference(mm->ioctx_table); + for (i = 0; i < table->nr; i++) { + struct kioctx *ctx; + + ctx = table->table[i]; + if (ctx && ctx->aio_ring_file == file) { + ctx->user_id = ctx->mmap_base = vma->vm_start; + break; + } + } + + rcu_read_unlock(); + spin_unlock(&mm->ioctx_lock); +} + static const struct file_operations aio_ring_fops = { .mmap = aio_ring_mmap, + .mremap = aio_ring_remap, }; #if IS_ENABLED(CONFIG_MIGRATION) diff --git a/include/linux/fs.h b/include/linux/fs.h index 9ab779e8a63c..85f378c55c26 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1497,6 +1497,7 @@ struct file_operations { long (*unlocked_ioctl) (struct file *, unsigned int, unsigned long); long (*compat_ioctl) (struct file *, unsigned int, unsigned long); int (*mmap) (struct file *, struct vm_area_struct *); + void (*mremap)(struct file *, struct vm_area_struct *); int (*open) (struct inode *, struct file *); int (*flush) (struct file *, fl_owner_t id); int (*release) (struct inode *, struct file *); diff --git a/mm/mremap.c b/mm/mremap.c index b147f66f4c40..c855922497a3 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -288,7 +288,8 @@ static unsigned long move_vma(struct vm_area_struct *vma, old_len = new_len; old_addr = new_addr; new_addr = -ENOMEM; - } + } else if (vma->vm_file && vma->vm_file->f_op->mremap) + vma->vm_file->f_op->mremap(vma->vm_file, new_vma); /* Conceal VM_ACCOUNT so old reservation is not undone */ if (vm_flags & VM_ACCOUNT) { -- cgit v1.2.3 From 5f785de588735306ec4d7c875caf9d28481c8b21 Mon Sep 17 00:00:00 2001 From: Fam Zheng Date: Thu, 6 Nov 2014 20:44:36 +0800 Subject: aio: Skip timer for io_getevents if timeout=0 In this case, it is basically a polling. Let's not involve timer at all because that would hurt performance for application event loops. In an arbitrary test I've done, io_getevents syscall elapsed time reduces from 50000+ nanoseconds to a few hundereds. Signed-off-by: Fam Zheng Signed-off-by: Benjamin LaHaise --- fs/aio.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/fs/aio.c b/fs/aio.c index bfab55607a4d..1b7893ecc296 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -1253,8 +1253,12 @@ static long read_events(struct kioctx *ctx, long min_nr, long nr, * the ringbuffer empty. So in practice we should be ok, but it's * something to be aware of when touching this code. */ - wait_event_interruptible_hrtimeout(ctx->wait, - aio_read_events(ctx, min_nr, nr, event, &ret), until); + if (until.tv64 == 0) + aio_read_events(ctx, min_nr, nr, event, &ret); + else + wait_event_interruptible_hrtimeout(ctx->wait, + aio_read_events(ctx, min_nr, nr, event, &ret), + until); if (!ret && signal_pending(current)) ret = -EINTR; -- cgit v1.2.3