summaryrefslogtreecommitdiffstats
path: root/mm
diff options
context:
space:
mode:
authorChristoph Lameter <cl@linux.com>2010-08-20 19:37:17 +0200
committerPekka Enberg <penberg@kernel.org>2010-10-02 09:24:27 +0200
commitc1d508365ea07f9f8849c7da5b02186601570f8b (patch)
treea087b2cda4546c51ef7ab4ba3c7764129a63e555 /mm
parentslub: Extract hooks for memory checkers from hotpaths (diff)
downloadlinux-c1d508365ea07f9f8849c7da5b02186601570f8b.tar.xz
linux-c1d508365ea07f9f8849c7da5b02186601570f8b.zip
slub: Move gfpflag masking out of the hotpath
Move the gfpflags masking into the hooks for checkers and into the slowpaths. gfpflag masking requires access to a global variable and thus adds an additional cacheline reference to the hotpaths. If no hooks are active then the gfpflag masking will result in code that the compiler can toss out. Acked-by: David Rientjes <rientjes@google.com> Signed-off-by: Christoph Lameter <cl@linux-foundation.org> Signed-off-by: Pekka Enberg <penberg@kernel.org>
Diffstat (limited to 'mm')
-rw-r--r--mm/slub.c5
1 files changed, 3 insertions, 2 deletions
diff --git a/mm/slub.c b/mm/slub.c
index ca49d02b5ff8..6608f2bc310b 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -796,6 +796,7 @@ static void trace(struct kmem_cache *s, struct page *page, void *object,
*/
static inline int slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags)
{
+ flags &= gfp_allowed_mask;
lockdep_trace_alloc(flags);
might_sleep_if(flags & __GFP_WAIT);
@@ -804,6 +805,7 @@ static inline int slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags)
static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags, void *object)
{
+ flags &= gfp_allowed_mask;
kmemcheck_slab_alloc(s, flags, object, s->objsize);
kmemleak_alloc_recursive(object, s->objsize, 1, s->flags, flags);
}
@@ -1677,6 +1679,7 @@ new_slab:
goto load_freelist;
}
+ gfpflags &= gfp_allowed_mask;
if (gfpflags & __GFP_WAIT)
local_irq_enable();
@@ -1725,8 +1728,6 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
struct kmem_cache_cpu *c;
unsigned long flags;
- gfpflags &= gfp_allowed_mask;
-
if (slab_pre_alloc_hook(s, gfpflags))
return NULL;