1 files changed, 149 insertions, 89 deletions

diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index f2d24eb3208c..6617ce4af095 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -71,9 +71,22 @@ union map_info *dm_get_mapinfo(struct bio *bio)
 #define DMF_DELETING 4
 #define DMF_NOFLUSH_SUSPENDING 5
 
+/*
+ * Work processed by per-device workqueue.
+ */
+struct dm_wq_req {
+	enum {
+		DM_WQ_FLUSH_ALL,
+		DM_WQ_FLUSH_DEFERRED,
+	} type;
+	struct work_struct work;
+	struct mapped_device *md;
+	void *context;
+};
+
 struct mapped_device {
 	struct rw_semaphore io_lock;
-	struct semaphore suspend_lock;
+	struct mutex suspend_lock;
 	spinlock_t pushback_lock;
 	rwlock_t map_lock;
 	atomic_t holders;
@@ -96,6 +109,11 @@ struct mapped_device {
 	struct bio_list pushback;
 
 	/*
+	 * Processing queue (flush/barriers)
+	 */
+	struct workqueue_struct *wq;
+
+	/*
 	 * The current mapping.
 	 */
 	struct dm_table *map;
@@ -181,7 +199,7 @@ static void local_exit(void)
 	DMINFO("cleaned up");
 }
 
-int (*_inits[])(void) __initdata = {
+static int (*_inits[])(void) __initdata = {
 	local_init,
 	dm_target_init,
 	dm_linear_init,
@@ -189,7 +207,7 @@ int (*_inits[])(void) __initdata = {
 	dm_interface_init,
 };
 
-void (*_exits[])(void) = {
+static void (*_exits[])(void) = {
 	local_exit,
 	dm_target_exit,
 	dm_linear_exit,
@@ -982,7 +1000,7 @@ static struct mapped_device *alloc_dev(int minor)
 	}
 
 	if (!try_module_get(THIS_MODULE))
-		goto bad0;
+		goto bad_module_get;
 
 	/* get a minor number for the dev */
 	if (minor == DM_ANY_MINOR)
@@ -990,11 +1008,11 @@ static struct mapped_device *alloc_dev(int minor)
 	else
 		r = specific_minor(md, minor);
 	if (r < 0)
-		goto bad1;
+		goto bad_minor;
 
 	memset(md, 0, sizeof(*md));
 	init_rwsem(&md->io_lock);
-	init_MUTEX(&md->suspend_lock);
+	mutex_init(&md->suspend_lock);
 	spin_lock_init(&md->pushback_lock);
 	rwlock_init(&md->map_lock);
 	atomic_set(&md->holders, 1);
@@ -1006,7 +1024,7 @@ static struct mapped_device *alloc_dev(int minor)
 
 	md->queue = blk_alloc_queue(GFP_KERNEL);
 	if (!md->queue)
-		goto bad1_free_minor;
+		goto bad_queue;
 
 	md->queue->queuedata = md;
 	md->queue->backing_dev_info.congested_fn = dm_any_congested;
@@ -1017,11 +1035,11 @@ static struct mapped_device *alloc_dev(int minor)
 
 	md->io_pool = mempool_create_slab_pool(MIN_IOS, _io_cache);
 	if (!md->io_pool)
-		goto bad2;
+		goto bad_io_pool;
 
 	md->tio_pool = mempool_create_slab_pool(MIN_IOS, _tio_cache);
 	if (!md->tio_pool)
-		goto bad3;
+		goto bad_tio_pool;
 
 	md->bs = bioset_create(16, 16);
 	if (!md->bs)
@@ -1029,7 +1047,7 @@ static struct mapped_device *alloc_dev(int minor)
 
 	md->disk = alloc_disk(1);
 	if (!md->disk)
-		goto bad4;
+		goto bad_disk;
 
 	atomic_set(&md->pending, 0);
 	init_waitqueue_head(&md->wait);
@@ -1044,6 +1062,10 @@ static struct mapped_device *alloc_dev(int minor)
 	add_disk(md->disk);
 	format_dev_t(md->name, MKDEV(_major, minor));
 
+	md->wq = create_singlethread_workqueue("kdmflush");
+	if (!md->wq)
+		goto bad_thread;
+
 	/* Populate the mapping, nobody knows we exist yet */
 	spin_lock(&_minor_lock);
 	old_md = idr_replace(&_minor_idr, md, minor);
@@ -1053,19 +1075,21 @@ static struct mapped_device *alloc_dev(int minor)
 
 	return md;
 
- bad4:
+bad_thread:
+	put_disk(md->disk);
+bad_disk:
 	bioset_free(md->bs);
- bad_no_bioset:
+bad_no_bioset:
 	mempool_destroy(md->tio_pool);
- bad3:
+bad_tio_pool:
 	mempool_destroy(md->io_pool);
- bad2:
+bad_io_pool:
 	blk_cleanup_queue(md->queue);
- bad1_free_minor:
+bad_queue:
 	free_minor(minor);
- bad1:
+bad_minor:
 	module_put(THIS_MODULE);
- bad0:
+bad_module_get:
 	kfree(md);
 	return NULL;
 }
@@ -1080,6 +1104,7 @@ static void free_dev(struct mapped_device *md)
 		unlock_fs(md);
 		bdput(md->suspended_bdev);
 	}
+	destroy_workqueue(md->wq);
 	mempool_destroy(md->tio_pool);
 	mempool_destroy(md->io_pool);
 	bioset_free(md->bs);
@@ -1259,20 +1284,91 @@ void dm_put(struct mapped_device *md)
 }
 EXPORT_SYMBOL_GPL(dm_put);
 
+static int dm_wait_for_completion(struct mapped_device *md)
+{
+	int r = 0;
+
+	while (1) {
+		set_current_state(TASK_INTERRUPTIBLE);
+
+		smp_mb();
+		if (!atomic_read(&md->pending))
+			break;
+
+		if (signal_pending(current)) {
+			r = -EINTR;
+			break;
+		}
+
+		io_schedule();
+	}
+	set_current_state(TASK_RUNNING);
+
+	return r;
+}
+
 /*
  * Process the deferred bios
  */
-static void __flush_deferred_io(struct mapped_device *md, struct bio *c)
+static void __flush_deferred_io(struct mapped_device *md)
 {
-	struct bio *n;
+	struct bio *c;
 
-	while (c) {
-		n = c->bi_next;
-		c->bi_next = NULL;
+	while ((c = bio_list_pop(&md->deferred))) {
 		if (__split_bio(md, c))
 			bio_io_error(c);
-		c = n;
 	}
+
+	clear_bit(DMF_BLOCK_IO, &md->flags);
+}
+
+static void __merge_pushback_list(struct mapped_device *md)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&md->pushback_lock, flags);
+	clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
+	bio_list_merge_head(&md->deferred, &md->pushback);
+	bio_list_init(&md->pushback);
+	spin_unlock_irqrestore(&md->pushback_lock, flags);
+}
+
+static void dm_wq_work(struct work_struct *work)
+{
+	struct dm_wq_req *req = container_of(work, struct dm_wq_req, work);
+	struct mapped_device *md = req->md;
+
+	down_write(&md->io_lock);
+	switch (req->type) {
+	case DM_WQ_FLUSH_ALL:
+		__merge_pushback_list(md);
+		/* pass through */
+	case DM_WQ_FLUSH_DEFERRED:
+		__flush_deferred_io(md);
+		break;
+	default:
+		DMERR("dm_wq_work: unrecognised work type %d", req->type);
+		BUG();
+	}
+	up_write(&md->io_lock);
+}
+
+static void dm_wq_queue(struct mapped_device *md, int type, void *context,
+			struct dm_wq_req *req)
+{
+	req->type = type;
+	req->md = md;
+	req->context = context;
+	INIT_WORK(&req->work, dm_wq_work);
+	queue_work(md->wq, &req->work);
+}
+
+static void dm_queue_flush(struct mapped_device *md, int type, void *context)
+{
+	struct dm_wq_req req;
+
+	dm_wq_queue(md, type, context, &req);
+	flush_workqueue(md->wq);
 }
 
 /*
@@ -1282,7 +1378,7 @@ int dm_swap_table(struct mapped_device *md, struct dm_table *table)
 {
 	int r = -EINVAL;
 
-	down(&md->suspend_lock);
+	mutex_lock(&md->suspend_lock);
 
 	/* device must be suspended */
 	if (!dm_suspended(md))
@@ -1297,7 +1393,7 @@ int dm_swap_table(struct mapped_device *md, struct dm_table *table)
 	r = __bind(md, table);
 
 out:
-	up(&md->suspend_lock);
+	mutex_unlock(&md->suspend_lock);
 	return r;
 }
 
@@ -1346,17 +1442,17 @@ static void unlock_fs(struct mapped_device *md)
 int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
 {
 	struct dm_table *map = NULL;
-	unsigned long flags;
 	DECLARE_WAITQUEUE(wait, current);
-	struct bio *def;
-	int r = -EINVAL;
+	int r = 0;
 	int do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG ? 1 : 0;
 	int noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG ? 1 : 0;
 
-	down(&md->suspend_lock);
+	mutex_lock(&md->suspend_lock);
 
-	if (dm_suspended(md))
+	if (dm_suspended(md)) {
+		r = -EINVAL;
 		goto out_unlock;
+	}
 
 	map = dm_get_table(md);
 
@@ -1378,16 +1474,16 @@ int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
 			r = -ENOMEM;
 			goto flush_and_out;
 		}
-	}
 
-	/*
-	 * Flush I/O to the device.
-	 * noflush supersedes do_lockfs, because lock_fs() needs to flush I/Os.
-	 */
-	if (do_lockfs && !noflush) {
-		r = lock_fs(md);
-		if (r)
-			goto out;
+		/*
+		 * Flush I/O to the device. noflush supersedes do_lockfs,
+		 * because lock_fs() needs to flush I/Os.
+		 */
+		if (do_lockfs) {
+			r = lock_fs(md);
+			if (r)
+				goto out;
+		}
 	}
 
 	/*
@@ -1404,66 +1500,36 @@ int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
 		dm_table_unplug_all(map);
 
 	/*
-	 * Then we wait for the already mapped ios to
-	 * complete.
+	 * Wait for the already-mapped ios to complete.
 	 */
-	while (1) {
-		set_current_state(TASK_INTERRUPTIBLE);
-
-		if (!atomic_read(&md->pending) || signal_pending(current))
-			break;
-
-		io_schedule();
-	}
-	set_current_state(TASK_RUNNING);
+	r = dm_wait_for_completion(md);
 
 	down_write(&md->io_lock);
 	remove_wait_queue(&md->wait, &wait);
 
-	if (noflush) {
-		spin_lock_irqsave(&md->pushback_lock, flags);
-		clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
-		bio_list_merge_head(&md->deferred, &md->pushback);
-		bio_list_init(&md->pushback);
-		spin_unlock_irqrestore(&md->pushback_lock, flags);
-	}
+	if (noflush)
+		__merge_pushback_list(md);
+	up_write(&md->io_lock);
 
 	/* were we interrupted ? */
-	r = -EINTR;
-	if (atomic_read(&md->pending)) {
-		clear_bit(DMF_BLOCK_IO, &md->flags);
-		def = bio_list_get(&md->deferred);
-		__flush_deferred_io(md, def);
-		up_write(&md->io_lock);
+	if (r < 0) {
+		dm_queue_flush(md, DM_WQ_FLUSH_DEFERRED, NULL);
+
 		unlock_fs(md);
 		goto out; /* pushback list is already flushed, so skip flush */
 	}
-	up_write(&md->io_lock);
 
 	dm_table_postsuspend_targets(map);
 
 	set_bit(DMF_SUSPENDED, &md->flags);
 
-	r = 0;
-
 flush_and_out:
-	if (r && noflush) {
+	if (r && noflush)
 		/*
 		 * Because there may be already I/Os in the pushback list,
 		 * flush them before return.
 		 */
-		down_write(&md->io_lock);
-
-		spin_lock_irqsave(&md->pushback_lock, flags);
-		clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
-		bio_list_merge_head(&md->deferred, &md->pushback);
-		bio_list_init(&md->pushback);
-		spin_unlock_irqrestore(&md->pushback_lock, flags);
-
-		def = bio_list_get(&md->deferred);
-		__flush_deferred_io(md, def);
-		up_write(&md->io_lock);
-	}
+		dm_queue_flush(md, DM_WQ_FLUSH_ALL, NULL);
 
 out:
 	if (r && md->suspended_bdev) {
@@ -1474,17 +1540,16 @@ out:
 	dm_table_put(map);
 
 out_unlock:
-	up(&md->suspend_lock);
+	mutex_unlock(&md->suspend_lock);
 	return r;
 }
 
 int dm_resume(struct mapped_device *md)
 {
 	int r = -EINVAL;
-	struct bio *def;
 	struct dm_table *map = NULL;
 
-	down(&md->suspend_lock);
+	mutex_lock(&md->suspend_lock);
 	if (!dm_suspended(md))
 		goto out;
 
@@ -1496,12 +1561,7 @@ int dm_resume(struct mapped_device *md)
 	if (r)
 		goto out;
 
-	down_write(&md->io_lock);
-	clear_bit(DMF_BLOCK_IO, &md->flags);
-
-	def = bio_list_get(&md->deferred);
-	__flush_deferred_io(md, def);
-	up_write(&md->io_lock);
+	dm_queue_flush(md, DM_WQ_FLUSH_DEFERRED, NULL);
 
 	unlock_fs(md);
 
@@ -1520,7 +1580,7 @@ int dm_resume(struct mapped_device *md)
 
 out:
 	dm_table_put(map);
-	up(&md->suspend_lock);
+	mutex_unlock(&md->suspend_lock);
 
 	return r;
 }