From d8d3d94b80aa1a1c0ca75c58b8abdc7356f38418 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 4 Mar 2014 21:27:34 -0500 Subject: pass iov_iter to ->direct_IO() unmodified, for now Signed-off-by: Al Viro --- fs/nfs/direct.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'fs/nfs/direct.c') diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c index b8797ae6831f..e9cde3935001 100644 --- a/fs/nfs/direct.c +++ b/fs/nfs/direct.c @@ -121,20 +121,20 @@ static inline int put_dreq(struct nfs_direct_req *dreq) * shunt off direct read and write requests before the VFS gets them, * so this method is only ever called for swap. */ -ssize_t nfs_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, loff_t pos, unsigned long nr_segs) +ssize_t nfs_direct_IO(int rw, struct kiocb *iocb, struct iov_iter *iter, loff_t pos) { #ifndef CONFIG_NFS_SWAP dprintk("NFS: nfs_direct_IO (%pD) off/no(%Ld/%lu) EINVAL\n", - iocb->ki_filp, (long long) pos, nr_segs); + iocb->ki_filp, (long long) pos, iter->nr_segs); return -EINVAL; #else VM_BUG_ON(iocb->ki_nbytes != PAGE_SIZE); if (rw == READ || rw == KERNEL_READ) - return nfs_file_direct_read(iocb, iov, nr_segs, pos, + return nfs_file_direct_read(iocb, iter->iov, iter->nr_segs, pos, rw == READ ? true : false); - return nfs_file_direct_write(iocb, iov, nr_segs, pos, + return nfs_file_direct_write(iocb, iter->iov, iter->nr_segs, pos, rw == WRITE ? true : false); #endif /* CONFIG_NFS_SWAP */ } -- cgit v1.2.3 From 619d30b4b8c488042b4a720ca79dccc346d1a516 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 4 Mar 2014 21:53:33 -0500 Subject: convert the guts of nfs_direct_IO() to iov_iter Signed-off-by: Al Viro --- fs/nfs/direct.c | 46 +++++++++++++++++++++------------------------- fs/nfs/file.c | 18 ++++++++++++------ include/linux/nfs_fs.h | 4 ++-- 3 files changed, 35 insertions(+), 33 deletions(-) (limited to 'fs/nfs/direct.c') diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c index e9cde3935001..21723149668b 100644 --- a/fs/nfs/direct.c +++ b/fs/nfs/direct.c @@ -132,9 +132,9 @@ ssize_t nfs_direct_IO(int rw, struct kiocb *iocb, struct iov_iter *iter, loff_t VM_BUG_ON(iocb->ki_nbytes != PAGE_SIZE); if (rw == READ || rw == KERNEL_READ) - return nfs_file_direct_read(iocb, iter->iov, iter->nr_segs, pos, + return nfs_file_direct_read(iocb, iter, pos, rw == READ ? true : false); - return nfs_file_direct_write(iocb, iter->iov, iter->nr_segs, pos, + return nfs_file_direct_write(iocb, iter, pos, rw == WRITE ? true : false); #endif /* CONFIG_NFS_SWAP */ } @@ -414,8 +414,7 @@ static ssize_t nfs_direct_read_schedule_segment(struct nfs_pageio_descriptor *de } static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq, - const struct iovec *iov, - unsigned long nr_segs, + struct iov_iter *iter, loff_t pos, bool uio) { struct nfs_pageio_descriptor desc; @@ -430,8 +429,8 @@ static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq, desc.pg_dreq = dreq; atomic_inc(&inode->i_dio_count); - for (seg = 0; seg < nr_segs; seg++) { - const struct iovec *vec = &iov[seg]; + for (seg = 0; seg < iter->nr_segs; seg++) { + const struct iovec *vec = &iter->iov[seg]; result = nfs_direct_read_schedule_segment(&desc, vec, pos, uio); if (result < 0) break; @@ -461,8 +460,7 @@ static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq, /** * nfs_file_direct_read - file direct read operation for NFS files * @iocb: target I/O control block - * @iov: vector of user buffers into which to read data - * @nr_segs: size of iov vector + * @iter: vector of user buffers into which to read data * @pos: byte offset in file where reading starts * * We use this function for direct reads instead of calling @@ -479,8 +477,8 @@ static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq, * client must read the updated atime from the server back into its * cache. */ -ssize_t nfs_file_direct_read(struct kiocb *iocb, const struct iovec *iov, - unsigned long nr_segs, loff_t pos, bool uio) +ssize_t nfs_file_direct_read(struct kiocb *iocb, struct iov_iter *iter, + loff_t pos, bool uio) { struct file *file = iocb->ki_filp; struct address_space *mapping = file->f_mapping; @@ -490,7 +488,7 @@ ssize_t nfs_file_direct_read(struct kiocb *iocb, const struct iovec *iov, ssize_t result = -EINVAL; size_t count; - count = iov_length(iov, nr_segs); + count = iov_length(iter->iov, iter->nr_segs); nfs_add_stats(mapping->host, NFSIOS_DIRECTREADBYTES, count); dfprintk(FILE, "NFS: direct read(%pD2, %zd@%Ld)\n", @@ -513,7 +511,7 @@ ssize_t nfs_file_direct_read(struct kiocb *iocb, const struct iovec *iov, goto out_unlock; dreq->inode = inode; - dreq->bytes_left = iov_length(iov, nr_segs); + dreq->bytes_left = count; dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp)); l_ctx = nfs_get_lock_context(dreq->ctx); if (IS_ERR(l_ctx)) { @@ -524,8 +522,8 @@ ssize_t nfs_file_direct_read(struct kiocb *iocb, const struct iovec *iov, if (!is_sync_kiocb(iocb)) dreq->iocb = iocb; - NFS_I(inode)->read_io += iov_length(iov, nr_segs); - result = nfs_direct_read_schedule_iovec(dreq, iov, nr_segs, pos, uio); + NFS_I(inode)->read_io += count; + result = nfs_direct_read_schedule_iovec(dreq, iter, pos, uio); mutex_unlock(&inode->i_mutex); @@ -864,8 +862,7 @@ static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops = { }; static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq, - const struct iovec *iov, - unsigned long nr_segs, + struct iov_iter *iter, loff_t pos, bool uio) { struct nfs_pageio_descriptor desc; @@ -880,9 +877,9 @@ static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq, get_dreq(dreq); atomic_inc(&inode->i_dio_count); - NFS_I(dreq->inode)->write_io += iov_length(iov, nr_segs); - for (seg = 0; seg < nr_segs; seg++) { - const struct iovec *vec = &iov[seg]; + NFS_I(dreq->inode)->write_io += iov_length(iter->iov, iter->nr_segs); + for (seg = 0; seg < iter->nr_segs; seg++) { + const struct iovec *vec = &iter->iov[seg]; result = nfs_direct_write_schedule_segment(&desc, vec, pos, uio); if (result < 0) break; @@ -911,8 +908,7 @@ static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq, /** * nfs_file_direct_write - file direct write operation for NFS files * @iocb: target I/O control block - * @iov: vector of user buffers from which to write data - * @nr_segs: size of iov vector + * @iter: vector of user buffers from which to write data * @pos: byte offset in file where writing starts * * We use this function for direct writes instead of calling @@ -930,8 +926,8 @@ static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq, * Note that O_APPEND is not supported for NFS direct writes, as there * is no atomic O_APPEND write facility in the NFS protocol. */ -ssize_t nfs_file_direct_write(struct kiocb *iocb, const struct iovec *iov, - unsigned long nr_segs, loff_t pos, bool uio) +ssize_t nfs_file_direct_write(struct kiocb *iocb, struct iov_iter *iter, + loff_t pos, bool uio) { ssize_t result = -EINVAL; struct file *file = iocb->ki_filp; @@ -942,7 +938,7 @@ ssize_t nfs_file_direct_write(struct kiocb *iocb, const struct iovec *iov, loff_t end; size_t count; - count = iov_length(iov, nr_segs); + count = iov_length(iter->iov, iter->nr_segs); end = (pos + count - 1) >> PAGE_CACHE_SHIFT; nfs_add_stats(mapping->host, NFSIOS_DIRECTWRITTENBYTES, count); @@ -993,7 +989,7 @@ ssize_t nfs_file_direct_write(struct kiocb *iocb, const struct iovec *iov, if (!is_sync_kiocb(iocb)) dreq->iocb = iocb; - result = nfs_direct_write_schedule_iovec(dreq, iov, nr_segs, pos, uio); + result = nfs_direct_write_schedule_iovec(dreq, iter, pos, uio); if (mapping->nrpages) { invalidate_inode_pages2_range(mapping, diff --git a/fs/nfs/file.c b/fs/nfs/file.c index 284ca901fe16..3d01b152894e 100644 --- a/fs/nfs/file.c +++ b/fs/nfs/file.c @@ -169,14 +169,18 @@ nfs_file_read(struct kiocb *iocb, const struct iovec *iov, unsigned long nr_segs, loff_t pos) { struct inode *inode = file_inode(iocb->ki_filp); + size_t count = iov_length(iov, nr_segs); ssize_t result; + struct iov_iter to; + + iov_iter_init(&to, iov, nr_segs, count, 0); if (iocb->ki_filp->f_flags & O_DIRECT) - return nfs_file_direct_read(iocb, iov, nr_segs, pos, true); + return nfs_file_direct_read(iocb, &to, pos, true); - dprintk("NFS: read(%pD2, %lu@%lu)\n", + dprintk("NFS: read(%pD2, %zu@%lu)\n", iocb->ki_filp, - (unsigned long) iov_length(iov, nr_segs), (unsigned long) pos); + count, (unsigned long) pos); result = nfs_revalidate_mapping(inode, iocb->ki_filp->f_mapping); if (!result) { @@ -643,16 +647,18 @@ ssize_t nfs_file_write(struct kiocb *iocb, const struct iovec *iov, unsigned long written = 0; ssize_t result; size_t count = iov_length(iov, nr_segs); + struct iov_iter from; + iov_iter_init(&from, iov, nr_segs, count, 0); result = nfs_key_timeout_notify(file, inode); if (result) return result; if (file->f_flags & O_DIRECT) - return nfs_file_direct_write(iocb, iov, nr_segs, pos, true); + return nfs_file_direct_write(iocb, &from, pos, true); - dprintk("NFS: write(%pD2, %lu@%Ld)\n", - file, (unsigned long) count, (long long) pos); + dprintk("NFS: write(%pD2, %zu@%Ld)\n", + file, count, (long long) pos); result = -EBUSY; if (IS_SWAPFILE(inode)) diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h index 5a0d78ec739d..0a82b6fbae8a 100644 --- a/include/linux/nfs_fs.h +++ b/include/linux/nfs_fs.h @@ -461,10 +461,10 @@ extern int nfs3_removexattr (struct dentry *, const char *name); */ extern ssize_t nfs_direct_IO(int, struct kiocb *, struct iov_iter *, loff_t); extern ssize_t nfs_file_direct_read(struct kiocb *iocb, - const struct iovec *iov, unsigned long nr_segs, + struct iov_iter *iter, loff_t pos, bool uio); extern ssize_t nfs_file_direct_write(struct kiocb *iocb, - const struct iovec *iov, unsigned long nr_segs, + struct iov_iter *iter, loff_t pos, bool uio); /* -- cgit v1.2.3 From a6cbcd4a4a85e2fdb0b3344b88df2e8b3d526b9e Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 4 Mar 2014 22:38:00 -0500 Subject: get rid of pointless iov_length() in ->direct_IO() all callers have iov_length(iter->iov, iter->nr_segs) == iov_iter_count(iter) Signed-off-by: Al Viro --- drivers/staging/lustre/lustre/llite/rw26.c | 2 +- fs/btrfs/inode.c | 2 +- fs/ext2/inode.c | 4 ++-- fs/ext3/inode.c | 2 +- fs/ext4/indirect.c | 2 +- fs/ext4/inode.c | 8 ++++---- fs/fat/inode.c | 6 +++--- fs/fuse/file.c | 2 +- fs/gfs2/aops.c | 2 +- fs/hfs/inode.c | 3 ++- fs/hfsplus/inode.c | 3 ++- fs/jfs/inode.c | 3 ++- fs/nfs/direct.c | 10 +++------- fs/nilfs2/inode.c | 3 ++- fs/reiserfs/inode.c | 3 ++- fs/udf/inode.c | 3 ++- fs/xfs/xfs_aops.c | 2 +- mm/filemap.c | 2 +- 18 files changed, 32 insertions(+), 30 deletions(-) (limited to 'fs/nfs/direct.c') diff --git a/drivers/staging/lustre/lustre/llite/rw26.c b/drivers/staging/lustre/lustre/llite/rw26.c index 66e05c6f4d27..38a5b580e7f0 100644 --- a/drivers/staging/lustre/lustre/llite/rw26.c +++ b/drivers/staging/lustre/lustre/llite/rw26.c @@ -370,7 +370,7 @@ static ssize_t ll_direct_IO_26(int rw, struct kiocb *iocb, struct file *file = iocb->ki_filp; struct inode *inode = file->f_mapping->host; struct ccc_object *obj = cl_inode2ccc(inode); - long count = iov_length(iter->iov, iter->nr_segs); + long count = iov_iter_count(iter); long tot_bytes = 0, result = 0; struct ll_inode_info *lli = ll_i2info(inode); unsigned long seg = 0; diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 30a6cc51f32c..c46a025d0c4b 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -7456,7 +7456,7 @@ static ssize_t btrfs_direct_IO(int rw, struct kiocb *iocb, * we need to flush the dirty pages again to make absolutely sure * that any outstanding dirty pages are on disk. */ - count = iov_length(iter->iov, iter->nr_segs); + count = iov_iter_count(iter); if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, &BTRFS_I(inode)->runtime_flags)) filemap_fdatawrite_range(inode->i_mapping, offset, count); diff --git a/fs/ext2/inode.c b/fs/ext2/inode.c index 47fbe760a7f8..116e809aa7cb 100644 --- a/fs/ext2/inode.c +++ b/fs/ext2/inode.c @@ -856,13 +856,13 @@ ext2_direct_IO(int rw, struct kiocb *iocb, struct iov_iter *iter, struct file *file = iocb->ki_filp; struct address_space *mapping = file->f_mapping; struct inode *inode = mapping->host; + size_t count = iov_iter_count(iter); ssize_t ret; ret = blockdev_direct_IO(rw, iocb, inode, iter->iov, offset, iter->nr_segs, ext2_get_block); if (ret < 0 && (rw & WRITE)) - ext2_write_failed(mapping, offset + - iov_length(iter->iov, iter->nr_segs)); + ext2_write_failed(mapping, offset + count); return ret; } diff --git a/fs/ext3/inode.c b/fs/ext3/inode.c index 7a5c501dc31b..8582ae2c80b0 100644 --- a/fs/ext3/inode.c +++ b/fs/ext3/inode.c @@ -1828,7 +1828,7 @@ static ssize_t ext3_direct_IO(int rw, struct kiocb *iocb, handle_t *handle; ssize_t ret; int orphan = 0; - size_t count = iov_length(iter->iov, iter->nr_segs); + size_t count = iov_iter_count(iter); int retries = 0; trace_ext3_direct_IO_enter(inode, offset, count, rw); diff --git a/fs/ext4/indirect.c b/fs/ext4/indirect.c index eb5ae16902d0..123898a6af05 100644 --- a/fs/ext4/indirect.c +++ b/fs/ext4/indirect.c @@ -647,7 +647,7 @@ ssize_t ext4_ind_direct_IO(int rw, struct kiocb *iocb, handle_t *handle; ssize_t ret; int orphan = 0; - size_t count = iov_length(iter->iov, iter->nr_segs); + size_t count = iov_iter_count(iter); int retries = 0; if (rw == WRITE) { diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 29996c1b673e..2b993579a968 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -3090,7 +3090,7 @@ static ssize_t ext4_ext_direct_IO(int rw, struct kiocb *iocb, struct file *file = iocb->ki_filp; struct inode *inode = file->f_mapping->host; ssize_t ret; - size_t count = iov_length(iter->iov, iter->nr_segs); + size_t count = iov_iter_count(iter); int overwrite = 0; get_block_t *get_block_func = NULL; int dio_flags = 0; @@ -3225,6 +3225,7 @@ static ssize_t ext4_direct_IO(int rw, struct kiocb *iocb, { struct file *file = iocb->ki_filp; struct inode *inode = file->f_mapping->host; + size_t count = iov_iter_count(iter); ssize_t ret; /* @@ -3237,13 +3238,12 @@ static ssize_t ext4_direct_IO(int rw, struct kiocb *iocb, if (ext4_has_inline_data(inode)) return 0; - trace_ext4_direct_IO_enter(inode, offset, iov_length(iter->iov, iter->nr_segs), rw); + trace_ext4_direct_IO_enter(inode, offset, count, rw); if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) ret = ext4_ext_direct_IO(rw, iocb, iter, offset); else ret = ext4_ind_direct_IO(rw, iocb, iter, offset); - trace_ext4_direct_IO_exit(inode, offset, - iov_length(iter->iov, iter->nr_segs), rw, ret); + trace_ext4_direct_IO_exit(inode, offset, count, rw, ret); return ret; } diff --git a/fs/fat/inode.c b/fs/fat/inode.c index d5237a199055..154a6f9d3189 100644 --- a/fs/fat/inode.c +++ b/fs/fat/inode.c @@ -191,6 +191,7 @@ static ssize_t fat_direct_IO(int rw, struct kiocb *iocb, struct file *file = iocb->ki_filp; struct address_space *mapping = file->f_mapping; struct inode *inode = mapping->host; + size_t count = iov_iter_count(iter); ssize_t ret; if (rw == WRITE) { @@ -203,7 +204,7 @@ static ssize_t fat_direct_IO(int rw, struct kiocb *iocb, * * Return 0, and fallback to normal buffered write. */ - loff_t size = offset + iov_length(iter->iov, iter->nr_segs); + loff_t size = offset + count; if (MSDOS_I(inode)->mmu_private < size) return 0; } @@ -215,8 +216,7 @@ static ssize_t fat_direct_IO(int rw, struct kiocb *iocb, ret = blockdev_direct_IO(rw, iocb, inode, iter->iov, offset, iter->nr_segs, fat_get_block); if (ret < 0 && (rw & WRITE)) - fat_write_failed(mapping, offset + - iov_length(iter->iov, iter->nr_segs)); + fat_write_failed(mapping, offset + count); return ret; } diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 17d96f36df15..36f15f9c1de2 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -2900,7 +2900,7 @@ fuse_direct_IO(int rw, struct kiocb *iocb, struct iov_iter *iter, loff_t pos = 0; struct inode *inode; loff_t i_size; - size_t count = iov_length(iter->iov, iter->nr_segs); + size_t count = iov_iter_count(iter); struct fuse_io_priv *io; pos = offset; diff --git a/fs/gfs2/aops.c b/fs/gfs2/aops.c index e84ddaa42104..228a12d2afa9 100644 --- a/fs/gfs2/aops.c +++ b/fs/gfs2/aops.c @@ -1081,7 +1081,7 @@ static ssize_t gfs2_direct_IO(int rw, struct kiocb *iocb, */ if (mapping->nrpages) { loff_t lstart = offset & (PAGE_CACHE_SIZE - 1); - loff_t len = iov_length(iter->iov, iter->nr_segs); + loff_t len = iov_iter_count(iter); loff_t end = PAGE_ALIGN(offset + len) - 1; rv = 0; diff --git a/fs/hfs/inode.c b/fs/hfs/inode.c index 09cff13528c5..dc69e8f31581 100644 --- a/fs/hfs/inode.c +++ b/fs/hfs/inode.c @@ -130,6 +130,7 @@ static ssize_t hfs_direct_IO(int rw, struct kiocb *iocb, struct file *file = iocb->ki_filp; struct address_space *mapping = file->f_mapping; struct inode *inode = file_inode(file)->i_mapping->host; + size_t count = iov_iter_count(iter); ssize_t ret; ret = blockdev_direct_IO(rw, iocb, inode, iter->iov, offset, @@ -141,7 +142,7 @@ static ssize_t hfs_direct_IO(int rw, struct kiocb *iocb, */ if (unlikely((rw & WRITE) && ret < 0)) { loff_t isize = i_size_read(inode); - loff_t end = offset + iov_length(iter->iov, iter->nr_segs); + loff_t end = offset + count; if (end > isize) hfs_write_failed(mapping, end); diff --git a/fs/hfsplus/inode.c b/fs/hfsplus/inode.c index 7f894a5b5eaf..e6b1251af47a 100644 --- a/fs/hfsplus/inode.c +++ b/fs/hfsplus/inode.c @@ -128,6 +128,7 @@ static ssize_t hfsplus_direct_IO(int rw, struct kiocb *iocb, struct file *file = iocb->ki_filp; struct address_space *mapping = file->f_mapping; struct inode *inode = file_inode(file)->i_mapping->host; + size_t count = iov_iter_count(iter); ssize_t ret; ret = blockdev_direct_IO(rw, iocb, inode, iter->iov, offset, iter->nr_segs, @@ -139,7 +140,7 @@ static ssize_t hfsplus_direct_IO(int rw, struct kiocb *iocb, */ if (unlikely((rw & WRITE) && ret < 0)) { loff_t isize = i_size_read(inode); - loff_t end = offset + iov_length(iter->iov, iter->nr_segs); + loff_t end = offset + count; if (end > isize) hfsplus_write_failed(mapping, end); diff --git a/fs/jfs/inode.c b/fs/jfs/inode.c index 7052744d5107..6cde5928693b 100644 --- a/fs/jfs/inode.c +++ b/fs/jfs/inode.c @@ -336,6 +336,7 @@ static ssize_t jfs_direct_IO(int rw, struct kiocb *iocb, struct file *file = iocb->ki_filp; struct address_space *mapping = file->f_mapping; struct inode *inode = file->f_mapping->host; + size_t count = iov_iter_count(iter); ssize_t ret; ret = blockdev_direct_IO(rw, iocb, inode, iter->iov, offset, @@ -347,7 +348,7 @@ static ssize_t jfs_direct_IO(int rw, struct kiocb *iocb, */ if (unlikely((rw & WRITE) && ret < 0)) { loff_t isize = i_size_read(inode); - loff_t end = offset + iov_length(iter->iov, iter->nr_segs); + loff_t end = offset + count; if (end > isize) jfs_write_failed(mapping, end); diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c index 21723149668b..1d34f454989e 100644 --- a/fs/nfs/direct.c +++ b/fs/nfs/direct.c @@ -486,9 +486,7 @@ ssize_t nfs_file_direct_read(struct kiocb *iocb, struct iov_iter *iter, struct nfs_direct_req *dreq; struct nfs_lock_context *l_ctx; ssize_t result = -EINVAL; - size_t count; - - count = iov_length(iter->iov, iter->nr_segs); + size_t count = iov_iter_count(iter); nfs_add_stats(mapping->host, NFSIOS_DIRECTREADBYTES, count); dfprintk(FILE, "NFS: direct read(%pD2, %zd@%Ld)\n", @@ -877,7 +875,7 @@ static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq, get_dreq(dreq); atomic_inc(&inode->i_dio_count); - NFS_I(dreq->inode)->write_io += iov_length(iter->iov, iter->nr_segs); + NFS_I(dreq->inode)->write_io += iov_iter_count(iter); for (seg = 0; seg < iter->nr_segs; seg++) { const struct iovec *vec = &iter->iov[seg]; result = nfs_direct_write_schedule_segment(&desc, vec, pos, uio); @@ -936,9 +934,7 @@ ssize_t nfs_file_direct_write(struct kiocb *iocb, struct iov_iter *iter, struct nfs_direct_req *dreq; struct nfs_lock_context *l_ctx; loff_t end; - size_t count; - - count = iov_length(iter->iov, iter->nr_segs); + size_t count = iov_iter_count(iter); end = (pos + count - 1) >> PAGE_CACHE_SHIFT; nfs_add_stats(mapping->host, NFSIOS_DIRECTWRITTENBYTES, count); diff --git a/fs/nilfs2/inode.c b/fs/nilfs2/inode.c index 1c0e8fedc095..7aaf913e8709 100644 --- a/fs/nilfs2/inode.c +++ b/fs/nilfs2/inode.c @@ -304,6 +304,7 @@ nilfs_direct_IO(int rw, struct kiocb *iocb, struct iov_iter *iter, struct file *file = iocb->ki_filp; struct address_space *mapping = file->f_mapping; struct inode *inode = file->f_mapping->host; + size_t count = iov_iter_count(iter); ssize_t size; if (rw == WRITE) @@ -319,7 +320,7 @@ nilfs_direct_IO(int rw, struct kiocb *iocb, struct iov_iter *iter, */ if (unlikely((rw & WRITE) && size < 0)) { loff_t isize = i_size_read(inode); - loff_t end = offset + iov_length(iter->iov, iter->nr_segs); + loff_t end = offset + count; if (end > isize) nilfs_write_failed(mapping, end); diff --git a/fs/reiserfs/inode.c b/fs/reiserfs/inode.c index 17bf4c41a509..723affe921f1 100644 --- a/fs/reiserfs/inode.c +++ b/fs/reiserfs/inode.c @@ -3087,6 +3087,7 @@ static ssize_t reiserfs_direct_IO(int rw, struct kiocb *iocb, { struct file *file = iocb->ki_filp; struct inode *inode = file->f_mapping->host; + size_t count = iov_iter_count(iter); ssize_t ret; ret = blockdev_direct_IO(rw, iocb, inode, iter->iov, offset, @@ -3098,7 +3099,7 @@ static ssize_t reiserfs_direct_IO(int rw, struct kiocb *iocb, */ if (unlikely((rw & WRITE) && ret < 0)) { loff_t isize = i_size_read(inode); - loff_t end = offset + iov_length(iter->iov, iter->nr_segs); + loff_t end = offset + count; if ((end > isize) && inode_newsize_ok(inode, isize) == 0) { truncate_setsize(inode, isize); diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 5b184c7f7dcb..28984baf6194 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -223,12 +223,13 @@ static ssize_t udf_direct_IO(int rw, struct kiocb *iocb, struct file *file = iocb->ki_filp; struct address_space *mapping = file->f_mapping; struct inode *inode = mapping->host; + size_t count = iov_iter_count(iter); ssize_t ret; ret = blockdev_direct_IO(rw, iocb, inode, iter->iov, offset, iter->nr_segs, udf_get_block); if (unlikely(ret < 0 && (rw & WRITE))) - udf_write_failed(mapping, offset + iov_length(iter->iov, iter->nr_segs)); + udf_write_failed(mapping, offset + count); return ret; } diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c index 330d7b1c4be3..6462b3186784 100644 --- a/fs/xfs/xfs_aops.c +++ b/fs/xfs/xfs_aops.c @@ -1458,7 +1458,7 @@ xfs_vm_direct_IO( ssize_t ret; if (rw & WRITE) { - size_t size = iov_length(iter->iov, iter->nr_segs); + size_t size = iov_iter_count(iter); /* * We cannot preallocate a size update transaction here as we diff --git a/mm/filemap.c b/mm/filemap.c index 139641274f1e..70c048ea36e0 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1698,7 +1698,7 @@ generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov, goto out; /* skip atime */ size = i_size_read(inode); retval = filemap_write_and_wait_range(mapping, pos, - pos + iov_length(iov, nr_segs) - 1); + pos + count - 1); if (!retval) retval = mapping->a_ops->direct_IO(READ, iocb, &i, pos); -- cgit v1.2.3 From 91f79c43d1b54d7154b118860d81b39bad07dfff Mon Sep 17 00:00:00 2001 From: Al Viro Date: Fri, 21 Mar 2014 04:58:33 -0400 Subject: new helper: iov_iter_get_pages_alloc() same as iov_iter_get_pages(), except that pages array is allocated (kmalloc if possible, vmalloc if that fails) and left for caller to free. Lustre and NFS ->direct_IO() switched to it. Signed-off-by: Al Viro --- drivers/staging/lustre/lustre/llite/rw26.c | 92 ++++----- fs/nfs/direct.c | 290 +++++++++-------------------- include/linux/uio.h | 2 + mm/iov_iter.c | 40 ++++ 4 files changed, 167 insertions(+), 257 deletions(-) (limited to 'fs/nfs/direct.c') diff --git a/drivers/staging/lustre/lustre/llite/rw26.c b/drivers/staging/lustre/lustre/llite/rw26.c index f718585c9e08..6b5994577b6b 100644 --- a/drivers/staging/lustre/lustre/llite/rw26.c +++ b/drivers/staging/lustre/lustre/llite/rw26.c @@ -218,14 +218,11 @@ static void ll_free_user_pages(struct page **pages, int npages, int do_dirty) int i; for (i = 0; i < npages; i++) { - if (pages[i] == NULL) - break; if (do_dirty) set_page_dirty_lock(pages[i]); page_cache_release(pages[i]); } - - OBD_FREE_LARGE(pages, npages * sizeof(*pages)); + kvfree(pages); } ssize_t ll_direct_rw_pages(const struct lu_env *env, struct cl_io *io, @@ -370,10 +367,9 @@ static ssize_t ll_direct_IO_26(int rw, struct kiocb *iocb, struct file *file = iocb->ki_filp; struct inode *inode = file->f_mapping->host; struct ccc_object *obj = cl_inode2ccc(inode); - long count = iov_iter_count(iter); - long tot_bytes = 0, result = 0; + ssize_t count = iov_iter_count(iter); + ssize_t tot_bytes = 0, result = 0; struct ll_inode_info *lli = ll_i2info(inode); - unsigned long seg = 0; long size = MAX_DIO_SIZE; int refcheck; @@ -407,63 +403,49 @@ static ssize_t ll_direct_IO_26(int rw, struct kiocb *iocb, mutex_lock(&inode->i_mutex); LASSERT(obj->cob_transient_pages == 0); - for (seg = 0; seg < iter->nr_segs; seg++) { - long iov_left = iter->iov[seg].iov_len; - unsigned long user_addr = (unsigned long)iter->iov[seg].iov_base; + while (iov_iter_count(iter)) { + struct page **pages; + size_t offs; + count = min_t(size_t, iov_iter_count(iter), size); if (rw == READ) { if (file_offset >= i_size_read(inode)) break; - if (file_offset + iov_left > i_size_read(inode)) - iov_left = i_size_read(inode) - file_offset; + if (file_offset + count > i_size_read(inode)) + count = i_size_read(inode) - file_offset; } - while (iov_left > 0) { - struct page **pages; - int page_count, max_pages = 0; - long bytes; - - bytes = min(size, iov_left); - page_count = ll_get_user_pages(rw, user_addr, bytes, - &pages, &max_pages); - if (likely(page_count > 0)) { - if (unlikely(page_count < max_pages)) - bytes = page_count << PAGE_CACHE_SHIFT; - result = ll_direct_IO_26_seg(env, io, rw, inode, - file->f_mapping, - bytes, file_offset, - pages, page_count); - ll_free_user_pages(pages, max_pages, rw==READ); - } else if (page_count == 0) { - GOTO(out, result = -EFAULT); - } else { - result = page_count; - } - if (unlikely(result <= 0)) { - /* If we can't allocate a large enough buffer - * for the request, shrink it to a smaller - * PAGE_SIZE multiple and try again. - * We should always be able to kmalloc for a - * page worth of page pointers = 4MB on i386. */ - if (result == -ENOMEM && - size > (PAGE_CACHE_SIZE / sizeof(*pages)) * - PAGE_CACHE_SIZE) { - size = ((((size / 2) - 1) | - ~CFS_PAGE_MASK) + 1) & - CFS_PAGE_MASK; - CDEBUG(D_VFSTRACE,"DIO size now %lu\n", - size); - continue; - } - - GOTO(out, result); + result = iov_iter_get_pages_alloc(iter, &pages, count, &offs); + if (likely(result > 0)) { + int n = (result + offs + PAGE_SIZE - 1) / PAGE_SIZE; + result = ll_direct_IO_26_seg(env, io, rw, inode, + file->f_mapping, + result, file_offset, + pages, n); + ll_free_user_pages(pages, n, rw==READ); + } + if (unlikely(result <= 0)) { + /* If we can't allocate a large enough buffer + * for the request, shrink it to a smaller + * PAGE_SIZE multiple and try again. + * We should always be able to kmalloc for a + * page worth of page pointers = 4MB on i386. */ + if (result == -ENOMEM && + size > (PAGE_CACHE_SIZE / sizeof(*pages)) * + PAGE_CACHE_SIZE) { + size = ((((size / 2) - 1) | + ~CFS_PAGE_MASK) + 1) & + CFS_PAGE_MASK; + CDEBUG(D_VFSTRACE,"DIO size now %lu\n", + size); + continue; } - tot_bytes += result; - file_offset += result; - iov_left -= result; - user_addr += result; + GOTO(out, result); } + iov_iter_advance(iter, result); + tot_bytes += result; + file_offset += result; } out: LASSERT(obj->cob_transient_pages == 0); diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c index 1d34f454989e..b122fe21fea0 100644 --- a/fs/nfs/direct.c +++ b/fs/nfs/direct.c @@ -322,60 +322,37 @@ static const struct nfs_pgio_completion_ops nfs_direct_read_completion_ops = { * handled automatically by nfs_direct_read_result(). Otherwise, if * no requests have been sent, just return an error. */ -static ssize_t nfs_direct_read_schedule_segment(struct nfs_pageio_descriptor *desc, - const struct iovec *iov, - loff_t pos, bool uio) -{ - struct nfs_direct_req *dreq = desc->pg_dreq; - struct nfs_open_context *ctx = dreq->ctx; - struct inode *inode = ctx->dentry->d_inode; - unsigned long user_addr = (unsigned long)iov->iov_base; - size_t count = iov->iov_len; - size_t rsize = NFS_SERVER(inode)->rsize; - unsigned int pgbase; - int result; - ssize_t started = 0; - struct page **pagevec = NULL; - unsigned int npages; - - do { - size_t bytes; - int i; - pgbase = user_addr & ~PAGE_MASK; - bytes = min(max_t(size_t, rsize, PAGE_SIZE), count); +static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq, + struct iov_iter *iter, + loff_t pos) +{ + struct nfs_pageio_descriptor desc; + struct inode *inode = dreq->inode; + ssize_t result = -EINVAL; + size_t requested_bytes = 0; + size_t rsize = max_t(size_t, NFS_SERVER(inode)->rsize, PAGE_SIZE); - result = -ENOMEM; - npages = nfs_page_array_len(pgbase, bytes); - if (!pagevec) - pagevec = kmalloc(npages * sizeof(struct page *), - GFP_KERNEL); - if (!pagevec) - break; - if (uio) { - down_read(¤t->mm->mmap_sem); - result = get_user_pages(current, current->mm, user_addr, - npages, 1, 0, pagevec, NULL); - up_read(¤t->mm->mmap_sem); - if (result < 0) - break; - } else { - WARN_ON(npages != 1); - result = get_kernel_page(user_addr, 1, pagevec); - if (WARN_ON(result != 1)) - break; - } + NFS_PROTO(dreq->inode)->read_pageio_init(&desc, dreq->inode, + &nfs_direct_read_completion_ops); + get_dreq(dreq); + desc.pg_dreq = dreq; + atomic_inc(&inode->i_dio_count); - if ((unsigned)result < npages) { - bytes = result * PAGE_SIZE; - if (bytes <= pgbase) { - nfs_direct_release_pages(pagevec, result); - break; - } - bytes -= pgbase; - npages = result; - } + while (iov_iter_count(iter)) { + struct page **pagevec; + size_t bytes; + size_t pgbase; + unsigned npages, i; + result = iov_iter_get_pages_alloc(iter, &pagevec, + rsize, &pgbase); + if (result < 0) + break; + + bytes = result; + iov_iter_advance(iter, bytes); + npages = (result + pgbase + PAGE_SIZE - 1) / PAGE_SIZE; for (i = 0; i < npages; i++) { struct nfs_page *req; unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase); @@ -389,55 +366,21 @@ static ssize_t nfs_direct_read_schedule_segment(struct nfs_pageio_descriptor *de } req->wb_index = pos >> PAGE_SHIFT; req->wb_offset = pos & ~PAGE_MASK; - if (!nfs_pageio_add_request(desc, req)) { - result = desc->pg_error; + if (!nfs_pageio_add_request(&desc, req)) { + result = desc.pg_error; nfs_release_request(req); break; } pgbase = 0; bytes -= req_len; - started += req_len; - user_addr += req_len; + requested_bytes += req_len; pos += req_len; - count -= req_len; dreq->bytes_left -= req_len; } - /* The nfs_page now hold references to these pages */ nfs_direct_release_pages(pagevec, npages); - } while (count != 0 && result >= 0); - - kfree(pagevec); - - if (started) - return started; - return result < 0 ? (ssize_t) result : -EFAULT; -} - -static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq, - struct iov_iter *iter, - loff_t pos, bool uio) -{ - struct nfs_pageio_descriptor desc; - struct inode *inode = dreq->inode; - ssize_t result = -EINVAL; - size_t requested_bytes = 0; - unsigned long seg; - - NFS_PROTO(dreq->inode)->read_pageio_init(&desc, dreq->inode, - &nfs_direct_read_completion_ops); - get_dreq(dreq); - desc.pg_dreq = dreq; - atomic_inc(&inode->i_dio_count); - - for (seg = 0; seg < iter->nr_segs; seg++) { - const struct iovec *vec = &iter->iov[seg]; - result = nfs_direct_read_schedule_segment(&desc, vec, pos, uio); + kvfree(pagevec); if (result < 0) break; - requested_bytes += result; - if ((size_t)result < vec->iov_len) - break; - pos += vec->iov_len; } nfs_pageio_complete(&desc); @@ -521,7 +464,7 @@ ssize_t nfs_file_direct_read(struct kiocb *iocb, struct iov_iter *iter, dreq->iocb = iocb; NFS_I(inode)->read_io += count; - result = nfs_direct_read_schedule_iovec(dreq, iter, pos, uio); + result = nfs_direct_read_schedule_iovec(dreq, iter, pos); mutex_unlock(&inode->i_mutex); @@ -677,109 +620,6 @@ static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode } #endif -/* - * NB: Return the value of the first error return code. Subsequent - * errors after the first one are ignored. - */ -/* - * For each wsize'd chunk of the user's buffer, dispatch an NFS WRITE - * operation. If nfs_writedata_alloc() or get_user_pages() fails, - * bail and stop sending more writes. Write length accounting is - * handled automatically by nfs_direct_write_result(). Otherwise, if - * no requests have been sent, just return an error. - */ -static ssize_t nfs_direct_write_schedule_segment(struct nfs_pageio_descriptor *desc, - const struct iovec *iov, - loff_t pos, bool uio) -{ - struct nfs_direct_req *dreq = desc->pg_dreq; - struct nfs_open_context *ctx = dreq->ctx; - struct inode *inode = ctx->dentry->d_inode; - unsigned long user_addr = (unsigned long)iov->iov_base; - size_t count = iov->iov_len; - size_t wsize = NFS_SERVER(inode)->wsize; - unsigned int pgbase; - int result; - ssize_t started = 0; - struct page **pagevec = NULL; - unsigned int npages; - - do { - size_t bytes; - int i; - - pgbase = user_addr & ~PAGE_MASK; - bytes = min(max_t(size_t, wsize, PAGE_SIZE), count); - - result = -ENOMEM; - npages = nfs_page_array_len(pgbase, bytes); - if (!pagevec) - pagevec = kmalloc(npages * sizeof(struct page *), GFP_KERNEL); - if (!pagevec) - break; - - if (uio) { - down_read(¤t->mm->mmap_sem); - result = get_user_pages(current, current->mm, user_addr, - npages, 0, 0, pagevec, NULL); - up_read(¤t->mm->mmap_sem); - if (result < 0) - break; - } else { - WARN_ON(npages != 1); - result = get_kernel_page(user_addr, 0, pagevec); - if (WARN_ON(result != 1)) - break; - } - - if ((unsigned)result < npages) { - bytes = result * PAGE_SIZE; - if (bytes <= pgbase) { - nfs_direct_release_pages(pagevec, result); - break; - } - bytes -= pgbase; - npages = result; - } - - for (i = 0; i < npages; i++) { - struct nfs_page *req; - unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase); - - req = nfs_create_request(dreq->ctx, dreq->inode, - pagevec[i], - pgbase, req_len); - if (IS_ERR(req)) { - result = PTR_ERR(req); - break; - } - nfs_lock_request(req); - req->wb_index = pos >> PAGE_SHIFT; - req->wb_offset = pos & ~PAGE_MASK; - if (!nfs_pageio_add_request(desc, req)) { - result = desc->pg_error; - nfs_unlock_and_release_request(req); - break; - } - pgbase = 0; - bytes -= req_len; - started += req_len; - user_addr += req_len; - pos += req_len; - count -= req_len; - dreq->bytes_left -= req_len; - } - /* The nfs_page now hold references to these pages */ - nfs_direct_release_pages(pagevec, npages); - } while (count != 0 && result >= 0); - - kfree(pagevec); - - if (started) - return started; - return result < 0 ? (ssize_t) result : -EFAULT; -} - static void nfs_direct_write_completion(struct nfs_pgio_header *hdr) { struct nfs_direct_req *dreq = hdr->dreq; @@ -859,15 +699,27 @@ static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops = { .completion = nfs_direct_write_completion, }; + +/* + * NB: Return the value of the first error return code. Subsequent + * errors after the first one are ignored. + */ +/* + * For each wsize'd chunk of the user's buffer, dispatch an NFS WRITE + * operation. If nfs_writedata_alloc() or get_user_pages() fails, + * bail and stop sending more writes. Write length accounting is + * handled automatically by nfs_direct_write_result(). Otherwise, if + * no requests have been sent, just return an error. + */ static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq, struct iov_iter *iter, - loff_t pos, bool uio) + loff_t pos) { struct nfs_pageio_descriptor desc; struct inode *inode = dreq->inode; ssize_t result = 0; size_t requested_bytes = 0; - unsigned long seg; + size_t wsize = max_t(size_t, NFS_SERVER(inode)->wsize, PAGE_SIZE); NFS_PROTO(inode)->write_pageio_init(&desc, inode, FLUSH_COND_STABLE, &nfs_direct_write_completion_ops); @@ -875,16 +727,50 @@ static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq, get_dreq(dreq); atomic_inc(&inode->i_dio_count); - NFS_I(dreq->inode)->write_io += iov_iter_count(iter); - for (seg = 0; seg < iter->nr_segs; seg++) { - const struct iovec *vec = &iter->iov[seg]; - result = nfs_direct_write_schedule_segment(&desc, vec, pos, uio); + NFS_I(inode)->write_io += iov_iter_count(iter); + while (iov_iter_count(iter)) { + struct page **pagevec; + size_t bytes; + size_t pgbase; + unsigned npages, i; + + result = iov_iter_get_pages_alloc(iter, &pagevec, + wsize, &pgbase); if (result < 0) break; - requested_bytes += result; - if ((size_t)result < vec->iov_len) + + bytes = result; + iov_iter_advance(iter, bytes); + npages = (result + pgbase + PAGE_SIZE - 1) / PAGE_SIZE; + for (i = 0; i < npages; i++) { + struct nfs_page *req; + unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase); + + req = nfs_create_request(dreq->ctx, inode, + pagevec[i], + pgbase, req_len); + if (IS_ERR(req)) { + result = PTR_ERR(req); + break; + } + nfs_lock_request(req); + req->wb_index = pos >> PAGE_SHIFT; + req->wb_offset = pos & ~PAGE_MASK; + if (!nfs_pageio_add_request(&desc, req)) { + result = desc.pg_error; + nfs_unlock_and_release_request(req); + break; + } + pgbase = 0; + bytes -= req_len; + requested_bytes += req_len; + pos += req_len; + dreq->bytes_left -= req_len; + } + nfs_direct_release_pages(pagevec, npages); + kvfree(pagevec); + if (result < 0) break; - pos += vec->iov_len; } nfs_pageio_complete(&desc); @@ -985,7 +871,7 @@ ssize_t nfs_file_direct_write(struct kiocb *iocb, struct iov_iter *iter, if (!is_sync_kiocb(iocb)) dreq->iocb = iocb; - result = nfs_direct_write_schedule_iovec(dreq, iter, pos, uio); + result = nfs_direct_write_schedule_iovec(dreq, iter, pos); if (mapping->nrpages) { invalidate_inode_pages2_range(mapping, diff --git a/include/linux/uio.h b/include/linux/uio.h index 2f8825b06680..4876e9f2a58f 100644 --- a/include/linux/uio.h +++ b/include/linux/uio.h @@ -73,6 +73,8 @@ void iov_iter_init(struct iov_iter *i, int direction, const struct iovec *iov, unsigned long nr_segs, size_t count); ssize_t iov_iter_get_pages(struct iov_iter *i, struct page **pages, size_t maxsize, size_t *start); +ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, struct page ***pages, + size_t maxsize, size_t *start); int iov_iter_npages(const struct iov_iter *i, int maxpages); static inline size_t iov_iter_count(struct iov_iter *i) diff --git a/mm/iov_iter.c b/mm/iov_iter.c index 0b677f8f9bad..a5c691c1a283 100644 --- a/mm/iov_iter.c +++ b/mm/iov_iter.c @@ -1,6 +1,8 @@ #include #include #include +#include +#include size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes, struct iov_iter *i) @@ -263,6 +265,44 @@ ssize_t iov_iter_get_pages(struct iov_iter *i, } EXPORT_SYMBOL(iov_iter_get_pages); +ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, + struct page ***pages, size_t maxsize, + size_t *start) +{ + size_t offset = i->iov_offset; + const struct iovec *iov = i->iov; + size_t len; + unsigned long addr; + void *p; + int n; + int res; + + len = iov->iov_len - offset; + if (len > i->count) + len = i->count; + if (len > maxsize) + len = maxsize; + addr = (unsigned long)iov->iov_base + offset; + len += *start = addr & (PAGE_SIZE - 1); + addr &= ~(PAGE_SIZE - 1); + n = (len + PAGE_SIZE - 1) / PAGE_SIZE; + + p = kmalloc(n * sizeof(struct page *), GFP_KERNEL); + if (!p) + p = vmalloc(n * sizeof(struct page *)); + if (!p) + return -ENOMEM; + + res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p); + if (unlikely(res < 0)) { + kvfree(p); + return res; + } + *pages = p; + return (res == n ? len : res * PAGE_SIZE) - *start; +} +EXPORT_SYMBOL(iov_iter_get_pages_alloc); + int iov_iter_npages(const struct iov_iter *i, int maxpages) { size_t offset = i->iov_offset; -- cgit v1.2.3