summaryrefslogtreecommitdiffstats
path: root/lib/sbitmap.c
diff options
context:
space:
mode:
authorOmar Sandoval <osandov@fb.com>2016-09-17 10:28:23 +0200
committerJens Axboe <axboe@fb.com>2016-09-17 16:39:10 +0200
commit40aabb67464d5aad9ca3d2a5fedee56e2ff45aa0 (patch)
tree19592825f8eb48363f5f9279c0d32003e7ed6532 /lib/sbitmap.c
parentsbitmap: allocate wait queues on a specific node (diff)
downloadlinux-40aabb67464d5aad9ca3d2a5fedee56e2ff45aa0.tar.xz
linux-40aabb67464d5aad9ca3d2a5fedee56e2ff45aa0.zip
sbitmap: push per-cpu last_tag into sbitmap_queue
Allocating your own per-cpu allocation hint separately makes for an awkward API. Instead, allocate the per-cpu hint as part of the struct sbitmap_queue. There's no point for a struct sbitmap_queue without the cache, but you can still use a bare struct sbitmap. Signed-off-by: Omar Sandoval <osandov@fb.com> Signed-off-by: Jens Axboe <axboe@fb.com>
Diffstat (limited to 'lib/sbitmap.c')
-rw-r--r--lib/sbitmap.c35
1 files changed, 34 insertions, 1 deletions
diff --git a/lib/sbitmap.c b/lib/sbitmap.c
index 4d8e97e470ee..1651ad9d5530 100644
--- a/lib/sbitmap.c
+++ b/lib/sbitmap.c
@@ -205,11 +205,18 @@ int sbitmap_queue_init_node(struct sbitmap_queue *sbq, unsigned int depth,
if (ret)
return ret;
+ sbq->alloc_hint = alloc_percpu_gfp(unsigned int, flags);
+ if (!sbq->alloc_hint) {
+ sbitmap_free(&sbq->sb);
+ return -ENOMEM;
+ }
+
sbq->wake_batch = sbq_calc_wake_batch(depth);
atomic_set(&sbq->wake_index, 0);
sbq->ws = kzalloc_node(SBQ_WAIT_QUEUES * sizeof(*sbq->ws), flags, node);
if (!sbq->ws) {
+ free_percpu(sbq->alloc_hint);
sbitmap_free(&sbq->sb);
return -ENOMEM;
}
@@ -229,6 +236,29 @@ void sbitmap_queue_resize(struct sbitmap_queue *sbq, unsigned int depth)
}
EXPORT_SYMBOL_GPL(sbitmap_queue_resize);
+int __sbitmap_queue_get(struct sbitmap_queue *sbq, bool round_robin)
+{
+ unsigned int hint;
+ int nr;
+
+ hint = this_cpu_read(*sbq->alloc_hint);
+ nr = sbitmap_get(&sbq->sb, hint, round_robin);
+
+ if (nr == -1) {
+ /* If the map is full, a hint won't do us much good. */
+ this_cpu_write(*sbq->alloc_hint, 0);
+ } else if (nr == hint || unlikely(round_robin)) {
+ /* Only update the hint if we used it. */
+ hint = nr + 1;
+ if (hint >= sbq->sb.depth - 1)
+ hint = 0;
+ this_cpu_write(*sbq->alloc_hint, hint);
+ }
+
+ return nr;
+}
+EXPORT_SYMBOL_GPL(__sbitmap_queue_get);
+
static struct sbq_wait_state *sbq_wake_ptr(struct sbitmap_queue *sbq)
{
int i, wake_index;
@@ -273,10 +303,13 @@ static void sbq_wake_up(struct sbitmap_queue *sbq)
}
}
-void sbitmap_queue_clear(struct sbitmap_queue *sbq, unsigned int nr)
+void sbitmap_queue_clear(struct sbitmap_queue *sbq, unsigned int nr,
+ bool round_robin, unsigned int cpu)
{
sbitmap_clear_bit(&sbq->sb, nr);
sbq_wake_up(sbq);
+ if (likely(!round_robin))
+ *per_cpu_ptr(sbq->alloc_hint, cpu) = nr;
}
EXPORT_SYMBOL_GPL(sbitmap_queue_clear);