dm bufio: conditionally enable branching for DM_BUFIO_CLIENT_NO_SLEEP

Use jump_label to limit the need for branching unless the optional
DM_BUFIO_CLIENT_NO_SLEEP is used.

Signed-off-by: Mike Snitzer <snitzer@kernel.org>

1 files changed, 11 insertions, 4 deletions

diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c
index 486179d1831c..0a38a7aef49c 100644
--- a/drivers/md/dm-bufio.c
+++ b/drivers/md/dm-bufio.c
@@ -18,6 +18,7 @@
 #include <linux/module.h>
 #include <linux/rbtree.h>
 #include <linux/stacktrace.h>
+#include <linux/jump_label.h>
 
 #define DM_MSG_PREFIX "bufio"
 
@@ -164,13 +165,15 @@ struct dm_buffer {
 #endif
 };
 
+static DEFINE_STATIC_KEY_FALSE(no_sleep_enabled);
+
 /*----------------------------------------------------------------*/
 
 #define dm_bufio_in_request()	(!!current->bio_list)
 
 static void dm_bufio_lock(struct dm_bufio_client *c)
 {
-	if (c->no_sleep)
+	if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep)
 		spin_lock_irqsave_nested(&c->spinlock, c->spinlock_flags, dm_bufio_in_request());
 	else
 		mutex_lock_nested(&c->lock, dm_bufio_in_request());
@@ -178,7 +181,7 @@ static void dm_bufio_lock(struct dm_bufio_client *c)
 
 static int dm_bufio_trylock(struct dm_bufio_client *c)
 {
-	if (c->no_sleep)
+	if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep)
 		return spin_trylock_irqsave(&c->spinlock, c->spinlock_flags);
 	else
 		return mutex_trylock(&c->lock);
@@ -186,7 +189,7 @@ static int dm_bufio_trylock(struct dm_bufio_client *c)
 
 static void dm_bufio_unlock(struct dm_bufio_client *c)
 {
-	if (c->no_sleep)
+	if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep)
 		spin_unlock_irqrestore(&c->spinlock, c->spinlock_flags);
 	else
 		mutex_unlock(&c->lock);
@@ -1760,8 +1763,10 @@ struct dm_bufio_client *dm_bufio_client_create(struct block_device *bdev, unsign
 	c->alloc_callback = alloc_callback;
 	c->write_callback = write_callback;
 
-	if (flags & DM_BUFIO_CLIENT_NO_SLEEP)
+	if (flags & DM_BUFIO_CLIENT_NO_SLEEP) {
 		c->no_sleep = true;
+		static_branch_inc(&no_sleep_enabled);
+	}
 
 	for (i = 0; i < LIST_SIZE; i++) {
 		INIT_LIST_HEAD(&c->lru[i]);
@@ -1895,6 +1900,8 @@ void dm_bufio_client_destroy(struct dm_bufio_client *c)
 	kmem_cache_destroy(c->slab_buffer);
 	dm_io_client_destroy(c->dm_io);
 	mutex_destroy(&c->lock);
+	if (c->no_sleep)
+		static_branch_dec(&no_sleep_enabled);
 	kfree(c);
 }
 EXPORT_SYMBOL_GPL(dm_bufio_client_destroy);