Btrfs: checksum file data at bio submission time instead of during writepage

When we checkum file data during writepage, the checksumming is done one
page at a time, making it difficult to do bulk metadata modifications
to insert checksums for large ranges of the file at once.

This patch changes btrfs to checksum on a per-bio basis instead.  The
bios are checksummed before they are handed off to the block layer, so
each bio is contiguous and only has pages from the same inode.

Checksumming on a bio basis allows us to insert and modify the file
checksum items in large groups.  It also allows the checksumming to
be done more easily by async worker threads.

Signed-off-by: Chris Mason <chris.mason@oracle.com>

5 files changed, 75 insertions, 18 deletions

diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
index 086cb0525cd1..92d892f92075 100644
--- a/fs/btrfs/ctree.h
+++ b/fs/btrfs/ctree.h
@@ -1142,11 +1142,9 @@ int btrfs_lookup_file_extent(struct btrfs_trans_handle *trans,
 			     struct btrfs_root *root,
 			     struct btrfs_path *path, u64 objectid,
 			     u64 bytenr, int mod);
-int btrfs_csum_file_block(struct btrfs_trans_handle *trans,
-			  struct btrfs_root *root,
-			  struct inode *inode,
-			  u64 objectid, u64 offset,
-			  char *data, size_t len);
+int btrfs_csum_file_blocks(struct btrfs_trans_handle *trans,
+			   struct btrfs_root *root, struct inode *inode,
+			   struct bio *bio);
 struct btrfs_csum_item *btrfs_lookup_csum(struct btrfs_trans_handle *trans,
 					  struct btrfs_root *root,
 					  struct btrfs_path *path,
diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index fb6400895ed6..e8130c876330 100644
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -1705,6 +1705,8 @@ static int submit_one_bio(int rw, struct bio *bio)
 			(unsigned long long)bio->bi_sector);
 		WARN_ON(1);
 	}
+	if (tree->ops && tree->ops->submit_bio_hook)
+		tree->ops->submit_bio_hook(rw, bio);
 
 	submit_bio(rw, bio);
 	if (bio_flagged(bio, BIO_EOPNOTSUPP))
diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h
index fcc4bb078c24..9d6654667089 100644
--- a/fs/btrfs/extent_io.h
+++ b/fs/btrfs/extent_io.h
@@ -28,6 +28,7 @@ struct extent_state;
 struct extent_io_ops {
 	int (*fill_delalloc)(struct inode *inode, u64 start, u64 end);
 	int (*writepage_io_hook)(struct page *page, u64 start, u64 end);
+	int (*submit_bio_hook)(int rw, struct bio *bio);
 	int (*readpage_io_hook)(struct page *page, u64 start, u64 end);
 	int (*readpage_end_io_hook)(struct page *page, u64 start, u64 end,
 				    struct extent_state *state);
diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c
index 3ebbc058d082..3f0e71b0e5d9 100644
--- a/fs/btrfs/file-item.c
+++ b/fs/btrfs/file-item.c
@@ -16,6 +16,9 @@
  * Boston, MA 021110-1307, USA.
  */
 
+#include <linux/bio.h>
+#include <linux/pagemap.h>
+#include <linux/highmem.h>
 #include "ctree.h"
 #include "disk-io.h"
 #include "transaction.h"
@@ -131,28 +134,35 @@ int btrfs_lookup_file_extent(struct btrfs_trans_handle *trans,
 	return ret;
 }
 
-int btrfs_csum_file_block(struct btrfs_trans_handle *trans,
-			  struct btrfs_root *root,
-			  struct inode *inode,
-			  u64 objectid, u64 offset,
-			  char *data, size_t len)
+int btrfs_csum_file_blocks(struct btrfs_trans_handle *trans,
+			   struct btrfs_root *root, struct inode *inode,
+			   struct bio *bio)
 {
+	u64 objectid = inode->i_ino;
+	u64 offset;
 	int ret;
 	struct btrfs_key file_key;
 	struct btrfs_key found_key;
-	u64 next_offset = (u64)-1;
-	int found_next = 0;
+	u64 next_offset;
+	int found_next;
 	struct btrfs_path *path;
 	struct btrfs_csum_item *item;
+	struct btrfs_csum_item *item_end;
 	struct extent_buffer *leaf = NULL;
 	u64 csum_offset;
-	u32 csum_result = ~(u32)0;
+	u32 csum_result;
 	u32 nritems;
 	u32 ins_size;
+	int bio_index = 0;
+	struct bio_vec *bvec = bio->bi_io_vec;
+	char *data;
 
 	path = btrfs_alloc_path();
 	BUG_ON(!path);
-
+again:
+	next_offset = (u64)-1;
+	found_next = 0;
+	offset = page_offset(bvec->bv_page) + bvec->bv_offset;
 	file_key.objectid = objectid;
 	file_key.offset = offset;
 	btrfs_set_key_type(&file_key, BTRFS_CSUM_ITEM_KEY);
@@ -259,7 +269,15 @@ csum:
 	item = (struct btrfs_csum_item *)((unsigned char *)item +
 					  csum_offset * BTRFS_CRC32_SIZE);
 found:
-	csum_result = btrfs_csum_data(root, data, csum_result, len);
+	item_end = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_csum_item);
+	item_end = (struct btrfs_csum_item *)((unsigned char *)item_end +
+				      btrfs_item_size_nr(leaf, path->slots[0]));
+next_bvec:
+	data = kmap_atomic(bvec->bv_page, KM_IRQ0);
+	csum_result = ~(u32)0;
+	csum_result = btrfs_csum_data(root, data + bvec->bv_offset,
+				      csum_result, bvec->bv_len);
+	kunmap_atomic(data, KM_IRQ0);
 	btrfs_csum_final(csum_result, (char *)&csum_result);
 	if (csum_result == 0) {
 		printk("csum result is 0 for inode %lu offset %Lu\n", inode->i_ino, offset);
@@ -267,9 +285,19 @@ found:
 
 	write_extent_buffer(leaf, &csum_result, (unsigned long)item,
 			    BTRFS_CRC32_SIZE);
+	bio_index++;
+	bvec++;
+	if (bio_index < bio->bi_vcnt) {
+		item = (struct btrfs_csum_item *)((char *)item + BTRFS_CRC32_SIZE);
+		if (item < item_end)
+			goto next_bvec;
+	}
 	btrfs_mark_buffer_dirty(path->nodes[0]);
+	if (bio_index < bio->bi_vcnt) {
+		btrfs_release_path(root, path);
+		goto again;
+	}
 fail:
-	btrfs_release_path(root, path);
 	btrfs_free_path(path);
 	return ret;
 }
diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
index 3e79572a5cb1..bbb71fad861a 100644
--- a/fs/btrfs/inode.c
+++ b/fs/btrfs/inode.c
@@ -16,6 +16,7 @@
  * Boston, MA 021110-1307, USA.
  */
 
+#include <linux/bio.h>
 #include <linux/buffer_head.h>
 #include <linux/fs.h>
 #include <linux/pagemap.h>
@@ -294,6 +295,32 @@ int btrfs_clear_bit_hook(struct inode *inode, u64 start, u64 end,
 	return 0;
 }
 
+int btrfs_submit_bio_hook(int rw, struct bio *bio)
+{
+	// struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
+	struct bio_vec *bvec = bio->bi_io_vec;
+	struct inode *inode = bvec->bv_page->mapping->host;
+	struct btrfs_root *root = BTRFS_I(inode)->root;
+	struct btrfs_trans_handle *trans;
+	int ret = 0;
+
+	if (rw != WRITE)
+		return 0;
+
+	if (btrfs_test_opt(root, NODATASUM) ||
+	    btrfs_test_flag(inode, NODATASUM))
+		return 0;
+
+	mutex_lock(&root->fs_info->fs_mutex);
+	trans = btrfs_start_transaction(root, 1);
+	btrfs_set_trans_block_group(trans, inode);
+	btrfs_csum_file_blocks(trans, root, inode, bio);
+	ret = btrfs_end_transaction(trans, root);
+	BUG_ON(ret);
+	mutex_unlock(&root->fs_info->fs_mutex);
+	return ret;
+}
+#if 0
 int btrfs_writepage_io_hook(struct page *page, u64 start, u64 end)
 {
 	struct inode *inode = page->mapping->host;
@@ -318,7 +345,7 @@ int btrfs_writepage_io_hook(struct page *page, u64 start, u64 end)
 	mutex_unlock(&root->fs_info->fs_mutex);
 	return ret;
 }
-
+#endif
 int btrfs_readpage_io_hook(struct page *page, u64 start, u64 end)
 {
 	int ret = 0;
@@ -3022,7 +3049,8 @@ static struct file_operations btrfs_dir_file_operations = {
 
 static struct extent_io_ops btrfs_extent_io_ops = {
 	.fill_delalloc = run_delalloc_range,
-	.writepage_io_hook = btrfs_writepage_io_hook,
+	// .writepage_io_hook = btrfs_writepage_io_hook,
+	.submit_bio_hook = btrfs_submit_bio_hook,
 	.readpage_io_hook = btrfs_readpage_io_hook,
 	.readpage_end_io_hook = btrfs_readpage_end_io_hook,
 	.set_bit_hook = btrfs_set_bit_hook,