From d9b7f22623b5fa9cc189581dcdfb2ac605933bf4 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Fri, 3 Aug 2012 22:51:37 +0400 Subject: [PATCH 01/21] slub: use free_page instead of put_page for freeing kmalloc allocation When freeing objects, the slub allocator will most of the time free empty pages by calling __free_pages(). But high-order kmalloc will be diposed by means of put_page() instead. It makes no sense to call put_page() in kernel pages that are provided by the object allocators, so we shouldn't be doing this ourselves. Aside from the consistency change, we don't change the flow too much. put_page()'s would call its dtor function, which is __free_pages. We also already do all of the Compound page tests ourselves, and the Mlock test we lose don't really matter. Signed-off-by: Glauber Costa Acked-by: Christoph Lameter CC: David Rientjes CC: Pekka Enberg Signed-off-by: Pekka Enberg --- mm/slub.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/slub.c b/mm/slub.c index 8f78e2577031..c83fe96f5e42 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -3477,7 +3477,7 @@ void kfree(const void *x) if (unlikely(!PageSlab(page))) { BUG_ON(!PageCompound(page)); kmemleak_free(x); - put_page(page); + __free_pages(page, compound_order(page)); return; } slab_free(page->slab, page, object, _RET_IP_); From 48f2474144ac708f1faad97e82a863ca8214b602 Mon Sep 17 00:00:00 2001 From: Michel Lespinasse Date: Tue, 14 Aug 2012 14:53:20 -0700 Subject: [PATCH 02/21] slab: do not call compound_head() in page_get_cache() page_get_cache() does not need to call compound_head(), as its unique caller virt_to_slab() already makes sure to return a head page. Additionally, removing the compound_head() call makes page_get_cache() consistent with page_get_slab(). Signed-off-by: Michel Lespinasse Cc: Christoph Lameter Cc: Pekka Enberg Acked-by: David Rientjes Signed-off-by: Andrew Morton Signed-off-by: Pekka Enberg --- mm/slab.c | 1 - 1 file changed, 1 deletion(-) diff --git a/mm/slab.c b/mm/slab.c index f8b0d539b482..cb6ce2dfc7cf 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -517,7 +517,6 @@ static bool slab_max_order_set __initdata; static inline struct kmem_cache *page_get_cache(struct page *page) { - page = compound_head(page); BUG_ON(!PageSlab(page)); return page->slab_cache; } From 455ce9eb1cfa083da0def023094190aeb133855a Mon Sep 17 00:00:00 2001 From: Andrew Morton Date: Tue, 14 Aug 2012 14:53:22 -0700 Subject: [PATCH 03/21] mm/slab_common.c: cleanup Eliminate an ifdef and a label by moving all the CONFIG_DEBUG_VM checking inside the locked region. Cc: Pekka Enberg Signed-off-by: Andrew Morton Signed-off-by: Pekka Enberg --- mm/slab_common.c | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/mm/slab_common.c b/mm/slab_common.c index aa3ca5bb01b5..281600b30100 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -53,19 +53,17 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align { struct kmem_cache *s = NULL; -#ifdef CONFIG_DEBUG_VM - if (!name || in_interrupt() || size < sizeof(void *) || - size > KMALLOC_MAX_SIZE) { - printk(KERN_ERR "kmem_cache_create(%s) integrity check" - " failed\n", name); - goto out; - } -#endif - get_online_cpus(); mutex_lock(&slab_mutex); #ifdef CONFIG_DEBUG_VM + if (!name || in_interrupt() || size < sizeof(void *) || + size > KMALLOC_MAX_SIZE) { + printk(KERN_ERR "kmem_cache_create(%s) integrity check" + " failed\n", name); + goto oops; + } + list_for_each_entry(s, &slab_caches, list) { char tmp; int res; @@ -104,9 +102,6 @@ oops: mutex_unlock(&slab_mutex); put_online_cpus(); -#ifdef CONFIG_DEBUG_VM -out: -#endif if (!s && (flags & SLAB_PANIC)) panic("kmem_cache_create: Failed to create slab '%s'\n", name); From 19c7ff9ecd89441096dab6a56f926f7df8ba850a Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Wed, 30 May 2012 12:54:46 -0500 Subject: [PATCH 04/21] slub: Take node lock during object free checks Only applies to scenarios where debugging is on: Validation of slabs can currently occur while debugging information is updated from the fast paths of the allocator. This results in various races where we get false reports about slab metadata not being in order. This patch makes the fast paths take the node lock so that serialization with slab validation will occur. Causes additional slowdown in debug scenarios. Reported-by: Waiman Long Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- mm/slub.c | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index c83fe96f5e42..e131084e87a3 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1069,13 +1069,13 @@ bad: return 0; } -static noinline int free_debug_processing(struct kmem_cache *s, - struct page *page, void *object, unsigned long addr) +static noinline struct kmem_cache_node *free_debug_processing( + struct kmem_cache *s, struct page *page, void *object, + unsigned long addr, unsigned long *flags) { - unsigned long flags; - int rc = 0; + struct kmem_cache_node *n = get_node(s, page_to_nid(page)); - local_irq_save(flags); + spin_lock_irqsave(&n->list_lock, *flags); slab_lock(page); if (!check_slab(s, page)) @@ -1113,15 +1113,19 @@ static noinline int free_debug_processing(struct kmem_cache *s, set_track(s, object, TRACK_FREE, addr); trace(s, page, object, 0); init_object(s, object, SLUB_RED_INACTIVE); - rc = 1; out: slab_unlock(page); - local_irq_restore(flags); - return rc; + /* + * Keep node_lock to preserve integrity + * until the object is actually freed + */ + return n; fail: + slab_unlock(page); + spin_unlock_irqrestore(&n->list_lock, *flags); slab_fix(s, "Object at 0x%p not freed", object); - goto out; + return NULL; } static int __init setup_slub_debug(char *str) @@ -1214,8 +1218,9 @@ static inline void setup_object_debug(struct kmem_cache *s, static inline int alloc_debug_processing(struct kmem_cache *s, struct page *page, void *object, unsigned long addr) { return 0; } -static inline int free_debug_processing(struct kmem_cache *s, - struct page *page, void *object, unsigned long addr) { return 0; } +static inline struct kmem_cache_node *free_debug_processing( + struct kmem_cache *s, struct page *page, void *object, + unsigned long addr, unsigned long *flags) { return NULL; } static inline int slab_pad_check(struct kmem_cache *s, struct page *page) { return 1; } @@ -2452,7 +2457,8 @@ static void __slab_free(struct kmem_cache *s, struct page *page, stat(s, FREE_SLOWPATH); - if (kmem_cache_debug(s) && !free_debug_processing(s, page, x, addr)) + if (kmem_cache_debug(s) && + !(n = free_debug_processing(s, page, x, addr, &flags))) return; do { From e24fc410f58cc7851188a6e996dc6ce5c4259eb4 Mon Sep 17 00:00:00 2001 From: Joonsoo Kim Date: Sat, 23 Jun 2012 03:22:38 +0900 Subject: [PATCH 05/21] slub: reduce failure of this_cpu_cmpxchg in put_cpu_partial() after unfreezing In current implementation, after unfreezing, we doesn't touch oldpage, so it remain 'NOT NULL'. When we call this_cpu_cmpxchg() with this old oldpage, this_cpu_cmpxchg() is mostly be failed. We can change value of oldpage to NULL after unfreezing, because unfreeze_partial() ensure that all the cpu partial slabs is removed from cpu partial list. In this time, we could expect that this_cpu_cmpxchg is mostly succeed. Acked-by: Christoph Lameter Signed-off-by: Joonsoo Kim Signed-off-by: Pekka Enberg --- mm/slub.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mm/slub.c b/mm/slub.c index e131084e87a3..c67bd0a4a952 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1962,6 +1962,7 @@ int put_cpu_partial(struct kmem_cache *s, struct page *page, int drain) local_irq_save(flags); unfreeze_partials(s); local_irq_restore(flags); + oldpage = NULL; pobjects = 0; pages = 0; stat(s, CPU_PARTIAL_DRAIN); From b920536aa0a4fd178b2957774adfe409027fe55b Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Thu, 16 Aug 2012 10:12:18 +0300 Subject: [PATCH 06/21] Revert "mm/slab_common.c: cleanup" This reverts commit 455ce9eb1cfa083da0def023094190aeb133855a. Andrew sent a better version. Signed-off-by: Pekka Enberg --- mm/slab_common.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/mm/slab_common.c b/mm/slab_common.c index 281600b30100..aa3ca5bb01b5 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -53,17 +53,19 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align { struct kmem_cache *s = NULL; - get_online_cpus(); - mutex_lock(&slab_mutex); - #ifdef CONFIG_DEBUG_VM if (!name || in_interrupt() || size < sizeof(void *) || size > KMALLOC_MAX_SIZE) { printk(KERN_ERR "kmem_cache_create(%s) integrity check" - " failed\n", name); - goto oops; + " failed\n", name); + goto out; } +#endif + get_online_cpus(); + mutex_lock(&slab_mutex); + +#ifdef CONFIG_DEBUG_VM list_for_each_entry(s, &slab_caches, list) { char tmp; int res; @@ -102,6 +104,9 @@ oops: mutex_unlock(&slab_mutex); put_online_cpus(); +#ifdef CONFIG_DEBUG_VM +out: +#endif if (!s && (flags & SLAB_PANIC)) panic("kmem_cache_create: Failed to create slab '%s'\n", name); From 77be4b1366c97b95bb197e18187ff45b0e4f6bd3 Mon Sep 17 00:00:00 2001 From: Shuah Khan Date: Thu, 16 Aug 2012 00:09:46 -0700 Subject: [PATCH 07/21] mm/slab: restructure kmem_cache_create() debug checks kmem_cache_create() does cache integrity checks when CONFIG_DEBUG_VM is defined. These checks interspersed with the regular code path has lead to compile time warnings when compiled without CONFIG_DEBUG_VM defined. Restructuring the code to move the integrity checks in to a new function would eliminate the current compile warning problem and also will allow for future changes to the debug only code to evolve without introducing new warnings in the regular path. This restructuring work is based on the discussion in the following thread: https://lkml.org/lkml/2012/7/13/424 [akpm@linux-foundation.org: fix build, cleanup] Signed-off-by: Shuah Khan Acked-by: Christoph Lameter Signed-off-by: Andrew Morton Signed-off-by: Pekka Enberg --- mm/slab_common.c | 97 ++++++++++++++++++++++++------------------------ 1 file changed, 48 insertions(+), 49 deletions(-) diff --git a/mm/slab_common.c b/mm/slab_common.c index aa3ca5bb01b5..8cf8b4962d6c 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -23,6 +23,52 @@ enum slab_state slab_state; LIST_HEAD(slab_caches); DEFINE_MUTEX(slab_mutex); +#ifdef CONFIG_DEBUG_VM +static int kmem_cache_sanity_check(const char *name, size_t size) +{ + struct kmem_cache *s = NULL; + + if (!name || in_interrupt() || size < sizeof(void *) || + size > KMALLOC_MAX_SIZE) { + pr_err("kmem_cache_create(%s) integrity check failed\n", name); + return -EINVAL; + } + + list_for_each_entry(s, &slab_caches, list) { + char tmp; + int res; + + /* + * This happens when the module gets unloaded and doesn't + * destroy its slab cache and no-one else reuses the vmalloc + * area of the module. Print a warning. + */ + res = probe_kernel_address(s->name, tmp); + if (res) { + pr_err("Slab cache with size %d has lost its name\n", + s->object_size); + continue; + } + + if (!strcmp(s->name, name)) { + pr_err("%s (%s): Cache name already exists.\n", + __func__, name); + dump_stack(); + s = NULL; + return -EINVAL; + } + } + + WARN_ON(strchr(name, ' ')); /* It confuses parsers */ + return 0; +} +#else +static inline int kmem_cache_sanity_check(const char *name, size_t size) +{ + return 0; +} +#endif + /* * kmem_cache_create - Create a cache. * @name: A string which is used in /proc/slabinfo to identify this cache. @@ -53,60 +99,13 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align { struct kmem_cache *s = NULL; -#ifdef CONFIG_DEBUG_VM - if (!name || in_interrupt() || size < sizeof(void *) || - size > KMALLOC_MAX_SIZE) { - printk(KERN_ERR "kmem_cache_create(%s) integrity check" - " failed\n", name); - goto out; - } -#endif - get_online_cpus(); mutex_lock(&slab_mutex); - -#ifdef CONFIG_DEBUG_VM - list_for_each_entry(s, &slab_caches, list) { - char tmp; - int res; - - /* - * This happens when the module gets unloaded and doesn't - * destroy its slab cache and no-one else reuses the vmalloc - * area of the module. Print a warning. - */ - res = probe_kernel_address(s->name, tmp); - if (res) { - printk(KERN_ERR - "Slab cache with size %d has lost its name\n", - s->object_size); - continue; - } - - if (!strcmp(s->name, name)) { - printk(KERN_ERR "kmem_cache_create(%s): Cache name" - " already exists.\n", - name); - dump_stack(); - s = NULL; - goto oops; - } - } - - WARN_ON(strchr(name, ' ')); /* It confuses parsers */ -#endif - - s = __kmem_cache_create(name, size, align, flags, ctor); - -#ifdef CONFIG_DEBUG_VM -oops: -#endif + if (kmem_cache_sanity_check(name, size) == 0) + s = __kmem_cache_create(name, size, align, flags, ctor); mutex_unlock(&slab_mutex); put_online_cpus(); -#ifdef CONFIG_DEBUG_VM -out: -#endif if (!s && (flags & SLAB_PANIC)) panic("kmem_cache_create: Failed to create slab '%s'\n", name); From c0931ef1f3e7305e55de1591602e5f25e4c11d1d Mon Sep 17 00:00:00 2001 From: David Rientjes Date: Thu, 16 Aug 2012 12:26:20 -0700 Subject: [PATCH 08/21] mm, slab: remove dflags cachep->dflags is never referenced, so remove it. Acked-by: Christoph Lameter Signed-off-by: David Rientjes Signed-off-by: Pekka Enberg --- include/linux/slab_def.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/linux/slab_def.h b/include/linux/slab_def.h index 0c634fa376c9..36d7031a1a53 100644 --- a/include/linux/slab_def.h +++ b/include/linux/slab_def.h @@ -45,7 +45,6 @@ struct kmem_cache { unsigned int colour_off; /* colour offset */ struct kmem_cache *slabp_cache; unsigned int slab_size; - unsigned int dflags; /* dynamic flags */ /* constructor func */ void (*ctor)(void *obj); From 5b74beb425e0eb009d16d4b9a0f55847a5f342fa Mon Sep 17 00:00:00 2001 From: David Rientjes Date: Thu, 16 Aug 2012 12:25:31 -0700 Subject: [PATCH 09/21] mm, slab: remove page_get_cache page_get_cache() isn't called from anything, so remove it. Signed-off-by: David Rientjes Signed-off-by: Pekka Enberg --- mm/slab.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/mm/slab.c b/mm/slab.c index cb6ce2dfc7cf..3b4587bb7b19 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -515,12 +515,6 @@ EXPORT_SYMBOL(slab_buffer_size); static int slab_max_order = SLAB_MAX_ORDER_LO; static bool slab_max_order_set __initdata; -static inline struct kmem_cache *page_get_cache(struct page *page) -{ - BUG_ON(!PageSlab(page)); - return page->slab_cache; -} - static inline struct kmem_cache *virt_to_cache(const void *obj) { struct page *page = virt_to_head_page(obj); From e21827aadd77e33833eeb393de2e8675e9b399e2 Mon Sep 17 00:00:00 2001 From: Ezequiel Garcia Date: Tue, 14 Aug 2012 09:55:21 -0300 Subject: [PATCH 10/21] mm: Use __do_krealloc to do the krealloc job Without this patch we can get (many) kmem trace events with call site at krealloc(). This happens because krealloc is calling __krealloc, which performs the allocation through kmalloc_track_caller. Since neither krealloc nor __krealloc are marked inline explicitly, the caller can be traced as being krealloc, which clearly is not the intended behavior. This patch allows to get the real caller of krealloc, by creating an always inlined function __do_krealloc, thus tracing the call site accurately. Acked-by: Christoph Lameter Cc: Glauber Costa Signed-off-by: Ezequiel Garcia Signed-off-by: Pekka Enberg --- mm/util.c | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/mm/util.c b/mm/util.c index 8c7265afa29f..dc3036cdcc6a 100644 --- a/mm/util.c +++ b/mm/util.c @@ -105,24 +105,12 @@ void *memdup_user(const void __user *src, size_t len) } EXPORT_SYMBOL(memdup_user); -/** - * __krealloc - like krealloc() but don't free @p. - * @p: object to reallocate memory for. - * @new_size: how many bytes of memory are required. - * @flags: the type of memory to allocate. - * - * This function is like krealloc() except it never frees the originally - * allocated buffer. Use this if you don't want to free the buffer immediately - * like, for example, with RCU. - */ -void *__krealloc(const void *p, size_t new_size, gfp_t flags) +static __always_inline void *__do_krealloc(const void *p, size_t new_size, + gfp_t flags) { void *ret; size_t ks = 0; - if (unlikely(!new_size)) - return ZERO_SIZE_PTR; - if (p) ks = ksize(p); @@ -135,6 +123,25 @@ void *__krealloc(const void *p, size_t new_size, gfp_t flags) return ret; } + +/** + * __krealloc - like krealloc() but don't free @p. + * @p: object to reallocate memory for. + * @new_size: how many bytes of memory are required. + * @flags: the type of memory to allocate. + * + * This function is like krealloc() except it never frees the originally + * allocated buffer. Use this if you don't want to free the buffer immediately + * like, for example, with RCU. + */ +void *__krealloc(const void *p, size_t new_size, gfp_t flags) +{ + if (unlikely(!new_size)) + return ZERO_SIZE_PTR; + + return __do_krealloc(p, new_size, flags); + +} EXPORT_SYMBOL(__krealloc); /** @@ -157,7 +164,7 @@ void *krealloc(const void *p, size_t new_size, gfp_t flags) return ZERO_SIZE_PTR; } - ret = __krealloc(p, new_size, flags); + ret = __do_krealloc(p, new_size, flags); if (ret && p != ret) kfree(p); From 947ca1856a7e60aa6d20536785e6a42dff25aa6e Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Wed, 5 Sep 2012 10:33:18 +0800 Subject: [PATCH 11/21] slab: fix the DEADLOCK issue on l3 alien lock DEADLOCK will be report while running a kernel with NUMA and LOCKDEP enabled, the process of this fake report is: kmem_cache_free() //free obj in cachep -> cache_free_alien() //acquire cachep's l3 alien lock -> __drain_alien_cache() -> free_block() -> slab_destroy() -> kmem_cache_free() //free slab in cachep->slabp_cache -> cache_free_alien() //acquire cachep->slabp_cache's l3 alien lock Since the cachep and cachep->slabp_cache's l3 alien are in the same lock class, fake report generated. This should not happen since we already have init_lock_keys() which will reassign the lock class for both l3 list and l3 alien. However, init_lock_keys() was invoked at a wrong position which is before we invoke enable_cpucache() on each cache. Since until set slab_state to be FULL, we won't invoke enable_cpucache() on caches to build their l3 alien while creating them, so although we invoked init_lock_keys(), the l3 alien lock class won't change since we don't have them until invoked enable_cpucache() later. This patch will invoke init_lock_keys() after we done enable_cpucache() instead of before to avoid the fake DEADLOCK report. Michael traced the problem back to a commit in release 3.0.0: commit 30765b92ada267c5395fc788623cb15233276f5c Author: Peter Zijlstra Date: Thu Jul 28 23:22:56 2011 +0200 slab, lockdep: Annotate the locks before using them Fernando found we hit the regular OFF_SLAB 'recursion' before we annotate the locks, cure this. The relevant portion of the stack-trace: > [ 0.000000] [] rt_spin_lock+0x50/0x56 > [ 0.000000] [] __cache_free+0x43/0xc3 > [ 0.000000] [] kmem_cache_free+0x6c/0xdc > [ 0.000000] [] slab_destroy+0x4f/0x53 > [ 0.000000] [] free_block+0x94/0xc1 > [ 0.000000] [] do_tune_cpucache+0x10b/0x2bb > [ 0.000000] [] enable_cpucache+0x7b/0xa7 > [ 0.000000] [] kmem_cache_init_late+0x1f/0x61 > [ 0.000000] [] start_kernel+0x24c/0x363 > [ 0.000000] [] i386_start_kernel+0xa9/0xaf Reported-by: Fernando Lopez-Lezcano Acked-by: Pekka Enberg Signed-off-by: Peter Zijlstra Link: http://lkml.kernel.org/r/1311888176.2617.379.camel@laptop Signed-off-by: Ingo Molnar The commit moved init_lock_keys() before we build up the alien, so we failed to reclass it. Cc: # 3.0+ Acked-by: Christoph Lameter Tested-by: Paul E. McKenney Signed-off-by: Michael Wang Signed-off-by: Pekka Enberg --- mm/slab.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm/slab.c b/mm/slab.c index 3b4587bb7b19..cd5a92650303 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -1774,9 +1774,6 @@ void __init kmem_cache_init_late(void) slab_state = UP; - /* Annotate slab for lockdep -- annotate the malloc caches */ - init_lock_keys(); - /* 6) resize the head arrays to their final sizes */ mutex_lock(&slab_mutex); list_for_each_entry(cachep, &slab_caches, list) @@ -1784,6 +1781,9 @@ void __init kmem_cache_init_late(void) BUG(); mutex_unlock(&slab_mutex); + /* Annotate slab for lockdep -- annotate the malloc caches */ + init_lock_keys(); + /* Done! */ slab_state = FULL; From 645df230cacc48f4463037016e9dbd3633183fe8 Mon Sep 17 00:00:00 2001 From: Dave Jones Date: Tue, 18 Sep 2012 15:54:12 -0400 Subject: [PATCH 12/21] mm, sl[au]b: Taint kernel when we detect a corrupted slab It doesn't seem worth adding a new taint flag for this, so just re-use the one from 'bad page' Acked-by: Christoph Lameter # SLUB Acked-by: David Rientjes Signed-off-by: Dave Jones Signed-off-by: Pekka Enberg --- mm/slab.c | 1 + mm/slub.c | 2 ++ 2 files changed, 3 insertions(+) diff --git a/mm/slab.c b/mm/slab.c index cd5a92650303..5c6abb831e6a 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -811,6 +811,7 @@ static void __slab_error(const char *function, struct kmem_cache *cachep, printk(KERN_ERR "slab error in %s(): cache `%s': %s\n", function, cachep->name, msg); dump_stack(); + add_taint(TAINT_BAD_PAGE); } /* diff --git a/mm/slub.c b/mm/slub.c index c67bd0a4a952..a6d043e13266 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -568,6 +568,8 @@ static void slab_bug(struct kmem_cache *s, char *fmt, ...) printk(KERN_ERR "BUG %s (%s): %s\n", s->name, print_tainted(), buf); printk(KERN_ERR "----------------------------------------" "-------------------------------------\n\n"); + + add_taint(TAINT_BAD_PAGE); } static void slab_fix(struct kmem_cache *s, char *fmt, ...) From 90f2cbbc49a8fe5a49cea1d362d90e377b949d49 Mon Sep 17 00:00:00 2001 From: Ezequiel Garcia Date: Sat, 8 Sep 2012 17:47:51 -0300 Subject: [PATCH 13/21] mm, slob: Use NUMA_NO_NODE instead of -1 Acked-by: David Rientjes Signed-off-by: Ezequiel Garcia Signed-off-by: Pekka Enberg --- include/linux/slob_def.h | 6 ++++-- mm/slob.c | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/include/linux/slob_def.h b/include/linux/slob_def.h index 0ec00b39d006..f28e14a12e3f 100644 --- a/include/linux/slob_def.h +++ b/include/linux/slob_def.h @@ -1,12 +1,14 @@ #ifndef __LINUX_SLOB_DEF_H #define __LINUX_SLOB_DEF_H +#include + void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node); static __always_inline void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) { - return kmem_cache_alloc_node(cachep, flags, -1); + return kmem_cache_alloc_node(cachep, flags, NUMA_NO_NODE); } void *__kmalloc_node(size_t size, gfp_t flags, int node); @@ -26,7 +28,7 @@ static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) */ static __always_inline void *kmalloc(size_t size, gfp_t flags) { - return __kmalloc_node(size, flags, -1); + return __kmalloc_node(size, flags, NUMA_NO_NODE); } static __always_inline void *__kmalloc(size_t size, gfp_t flags) diff --git a/mm/slob.c b/mm/slob.c index 45d4ca79933a..191e1713a6d9 100644 --- a/mm/slob.c +++ b/mm/slob.c @@ -194,7 +194,7 @@ static void *slob_new_pages(gfp_t gfp, int order, int node) void *page; #ifdef CONFIG_NUMA - if (node != -1) + if (node != NUMA_NO_NODE) page = alloc_pages_exact_node(node, gfp, order); else #endif @@ -290,7 +290,7 @@ static void *slob_alloc(size_t size, gfp_t gfp, int align, int node) * If there's a node specification, search for a partial * page with a matching node id in the freelist. */ - if (node != -1 && page_to_nid(sp) != node) + if (node != NUMA_NO_NODE && page_to_nid(sp) != node) continue; #endif /* Enough room on this page? */ @@ -514,7 +514,7 @@ struct kmem_cache *__kmem_cache_create(const char *name, size_t size, struct kmem_cache *c; c = slob_alloc(sizeof(struct kmem_cache), - GFP_KERNEL, ARCH_KMALLOC_MINALIGN, -1); + GFP_KERNEL, ARCH_KMALLOC_MINALIGN, NUMA_NO_NODE); if (c) { c->name = name; From ff4fcd01ec86d98d15d2fd96f22f19bb1d341b88 Mon Sep 17 00:00:00 2001 From: Ezequiel Garcia Date: Sat, 8 Sep 2012 17:47:52 -0300 Subject: [PATCH 14/21] mm, slab: Remove silly function slab_buffer_size() This function is seldom used, and can be simply replaced with cachep->size. Acked-by: David Rientjes Signed-off-by: Ezequiel Garcia Signed-off-by: Pekka Enberg --- include/linux/slab_def.h | 5 ----- mm/slab.c | 12 ++---------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/include/linux/slab_def.h b/include/linux/slab_def.h index 36d7031a1a53..604ebc832a9a 100644 --- a/include/linux/slab_def.h +++ b/include/linux/slab_def.h @@ -113,17 +113,12 @@ void *__kmalloc(size_t size, gfp_t flags); #ifdef CONFIG_TRACING extern void *kmem_cache_alloc_trace(size_t size, struct kmem_cache *cachep, gfp_t flags); -extern size_t slab_buffer_size(struct kmem_cache *cachep); #else static __always_inline void * kmem_cache_alloc_trace(size_t size, struct kmem_cache *cachep, gfp_t flags) { return kmem_cache_alloc(cachep, flags); } -static inline size_t slab_buffer_size(struct kmem_cache *cachep) -{ - return 0; -} #endif static __always_inline void *kmalloc(size_t size, gfp_t flags) diff --git a/mm/slab.c b/mm/slab.c index 5c6abb831e6a..7072848701e0 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -498,14 +498,6 @@ static void **dbg_userword(struct kmem_cache *cachep, void *objp) #endif -#ifdef CONFIG_TRACING -size_t slab_buffer_size(struct kmem_cache *cachep) -{ - return cachep->size; -} -EXPORT_SYMBOL(slab_buffer_size); -#endif - /* * Do not go above this order unless 0 objects fit into the slab or * overridden on the command line. @@ -3850,7 +3842,7 @@ kmem_cache_alloc_trace(size_t size, struct kmem_cache *cachep, gfp_t flags) ret = __cache_alloc(cachep, flags, __builtin_return_address(0)); trace_kmalloc(_RET_IP_, ret, - size, slab_buffer_size(cachep), flags); + size, cachep->size, flags); return ret; } EXPORT_SYMBOL(kmem_cache_alloc_trace); @@ -3881,7 +3873,7 @@ void *kmem_cache_alloc_node_trace(size_t size, ret = __cache_alloc_node(cachep, flags, nodeid, __builtin_return_address(0)); trace_kmalloc_node(_RET_IP_, ret, - size, slab_buffer_size(cachep), + size, cachep->size, flags, nodeid); return ret; } From f3f741019595f1e73564d985f5fe8abcbb98c769 Mon Sep 17 00:00:00 2001 From: Ezequiel Garcia Date: Sat, 8 Sep 2012 17:47:53 -0300 Subject: [PATCH 15/21] mm, slob: Add support for kmalloc_track_caller() Currently slob falls back to regular kmalloc for this case. With this patch kmalloc_track_caller() is correctly implemented, thus tracing the specified caller. This is important to trace accurately allocations performed by krealloc, kstrdup, kmemdup, etc. Signed-off-by: Ezequiel Garcia Signed-off-by: Pekka Enberg --- include/linux/slab.h | 6 ++++-- mm/slob.c | 27 ++++++++++++++++++++++++--- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/include/linux/slab.h b/include/linux/slab.h index 0dd2dfa7beca..83d1a1454b7e 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -321,7 +321,8 @@ static inline void *kmem_cache_alloc_node(struct kmem_cache *cachep, * request comes from. */ #if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_SLUB) || \ - (defined(CONFIG_SLAB) && defined(CONFIG_TRACING)) + (defined(CONFIG_SLAB) && defined(CONFIG_TRACING)) || \ + (defined(CONFIG_SLOB) && defined(CONFIG_TRACING)) extern void *__kmalloc_track_caller(size_t, gfp_t, unsigned long); #define kmalloc_track_caller(size, flags) \ __kmalloc_track_caller(size, flags, _RET_IP_) @@ -340,7 +341,8 @@ extern void *__kmalloc_track_caller(size_t, gfp_t, unsigned long); * allocation request comes from. */ #if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_SLUB) || \ - (defined(CONFIG_SLAB) && defined(CONFIG_TRACING)) + (defined(CONFIG_SLAB) && defined(CONFIG_TRACING)) || \ + (defined(CONFIG_SLOB) && defined(CONFIG_TRACING)) extern void *__kmalloc_node_track_caller(size_t, gfp_t, int, unsigned long); #define kmalloc_node_track_caller(size, flags, node) \ __kmalloc_node_track_caller(size, flags, node, \ diff --git a/mm/slob.c b/mm/slob.c index 191e1713a6d9..dd47d16d57b6 100644 --- a/mm/slob.c +++ b/mm/slob.c @@ -425,7 +425,8 @@ out: * End of slob allocator proper. Begin kmem_cache_alloc and kmalloc frontend. */ -void *__kmalloc_node(size_t size, gfp_t gfp, int node) +static __always_inline void * +__do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller) { unsigned int *m; int align = max(ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN); @@ -446,7 +447,7 @@ void *__kmalloc_node(size_t size, gfp_t gfp, int node) *m = size; ret = (void *)m + align; - trace_kmalloc_node(_RET_IP_, ret, + trace_kmalloc_node(caller, ret, size, size + align, gfp, node); } else { unsigned int order = get_order(size); @@ -460,15 +461,35 @@ void *__kmalloc_node(size_t size, gfp_t gfp, int node) page->private = size; } - trace_kmalloc_node(_RET_IP_, ret, + trace_kmalloc_node(caller, ret, size, PAGE_SIZE << order, gfp, node); } kmemleak_alloc(ret, size, 1, gfp); return ret; } + +void *__kmalloc_node(size_t size, gfp_t gfp, int node) +{ + return __do_kmalloc_node(size, gfp, node, _RET_IP_); +} EXPORT_SYMBOL(__kmalloc_node); +#ifdef CONFIG_TRACING +void *__kmalloc_track_caller(size_t size, gfp_t gfp, unsigned long caller) +{ + return __do_kmalloc_node(size, gfp, NUMA_NO_NODE, caller); +} + +#ifdef CONFIG_NUMA +void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags, + int node, unsigned long caller) +{ + return __do_kmalloc_node(size, gfp, node, caller); +} +#endif +#endif + void kfree(const void *block) { struct page *sp; From 7c0cb9c64f83dd9bf18b139ac0de00daba6087c2 Mon Sep 17 00:00:00 2001 From: Ezequiel Garcia Date: Sat, 8 Sep 2012 17:47:55 -0300 Subject: [PATCH 16/21] mm, slab: Replace 'caller' type, void* -> unsigned long This allows to use _RET_IP_ instead of builtin_address(0), thus achiveing implementation consistency in all three allocators. Though maybe a nitpick, the real goal behind this patch is to be able to obtain common code between SLAB and SLUB. Signed-off-by: Ezequiel Garcia Signed-off-by: Pekka Enberg --- mm/slab.c | 50 ++++++++++++++++++++++++-------------------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/mm/slab.c b/mm/slab.c index 7072848701e0..6c356793612d 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -3084,7 +3084,7 @@ static inline void verify_redzone_free(struct kmem_cache *cache, void *obj) } static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp, - void *caller) + unsigned long caller) { struct page *page; unsigned int objnr; @@ -3104,7 +3104,7 @@ static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp, *dbg_redzone2(cachep, objp) = RED_INACTIVE; } if (cachep->flags & SLAB_STORE_USER) - *dbg_userword(cachep, objp) = caller; + *dbg_userword(cachep, objp) = (void *)caller; objnr = obj_to_index(cachep, slabp, objp); @@ -3117,7 +3117,7 @@ static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp, if (cachep->flags & SLAB_POISON) { #ifdef CONFIG_DEBUG_PAGEALLOC if ((cachep->size % PAGE_SIZE)==0 && OFF_SLAB(cachep)) { - store_stackinfo(cachep, objp, (unsigned long)caller); + store_stackinfo(cachep, objp, caller); kernel_map_pages(virt_to_page(objp), cachep->size / PAGE_SIZE, 0); } else { @@ -3270,7 +3270,7 @@ static inline void cache_alloc_debugcheck_before(struct kmem_cache *cachep, #if DEBUG static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep, - gfp_t flags, void *objp, void *caller) + gfp_t flags, void *objp, unsigned long caller) { if (!objp) return objp; @@ -3287,7 +3287,7 @@ static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep, poison_obj(cachep, objp, POISON_INUSE); } if (cachep->flags & SLAB_STORE_USER) - *dbg_userword(cachep, objp) = caller; + *dbg_userword(cachep, objp) = (void *)caller; if (cachep->flags & SLAB_RED_ZONE) { if (*dbg_redzone1(cachep, objp) != RED_INACTIVE || @@ -3562,7 +3562,7 @@ done: */ static __always_inline void * __cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, - void *caller) + unsigned long caller) { unsigned long save_flags; void *ptr; @@ -3648,7 +3648,7 @@ __do_cache_alloc(struct kmem_cache *cachep, gfp_t flags) #endif /* CONFIG_NUMA */ static __always_inline void * -__cache_alloc(struct kmem_cache *cachep, gfp_t flags, void *caller) +__cache_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller) { unsigned long save_flags; void *objp; @@ -3784,7 +3784,7 @@ free_done: * be in this state _before_ it is released. Called with disabled ints. */ static inline void __cache_free(struct kmem_cache *cachep, void *objp, - void *caller) + unsigned long caller) { struct array_cache *ac = cpu_cache_get(cachep); @@ -3824,7 +3824,7 @@ static inline void __cache_free(struct kmem_cache *cachep, void *objp, */ void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) { - void *ret = __cache_alloc(cachep, flags, __builtin_return_address(0)); + void *ret = __cache_alloc(cachep, flags, _RET_IP_); trace_kmem_cache_alloc(_RET_IP_, ret, cachep->object_size, cachep->size, flags); @@ -3839,7 +3839,7 @@ kmem_cache_alloc_trace(size_t size, struct kmem_cache *cachep, gfp_t flags) { void *ret; - ret = __cache_alloc(cachep, flags, __builtin_return_address(0)); + ret = __cache_alloc(cachep, flags, _RET_IP_); trace_kmalloc(_RET_IP_, ret, size, cachep->size, flags); @@ -3851,8 +3851,7 @@ EXPORT_SYMBOL(kmem_cache_alloc_trace); #ifdef CONFIG_NUMA void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid) { - void *ret = __cache_alloc_node(cachep, flags, nodeid, - __builtin_return_address(0)); + void *ret = __cache_alloc_node(cachep, flags, nodeid, _RET_IP_); trace_kmem_cache_alloc_node(_RET_IP_, ret, cachep->object_size, cachep->size, @@ -3870,8 +3869,8 @@ void *kmem_cache_alloc_node_trace(size_t size, { void *ret; - ret = __cache_alloc_node(cachep, flags, nodeid, - __builtin_return_address(0)); + ret = __cache_alloc_node(cachep, flags, nodeid, _RET_IP); + trace_kmalloc_node(_RET_IP_, ret, size, cachep->size, flags, nodeid); @@ -3881,7 +3880,7 @@ EXPORT_SYMBOL(kmem_cache_alloc_node_trace); #endif static __always_inline void * -__do_kmalloc_node(size_t size, gfp_t flags, int node, void *caller) +__do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller) { struct kmem_cache *cachep; @@ -3894,21 +3893,20 @@ __do_kmalloc_node(size_t size, gfp_t flags, int node, void *caller) #if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_TRACING) void *__kmalloc_node(size_t size, gfp_t flags, int node) { - return __do_kmalloc_node(size, flags, node, - __builtin_return_address(0)); + return __do_kmalloc_node(size, flags, node, _RET_IP_); } EXPORT_SYMBOL(__kmalloc_node); void *__kmalloc_node_track_caller(size_t size, gfp_t flags, int node, unsigned long caller) { - return __do_kmalloc_node(size, flags, node, (void *)caller); + return __do_kmalloc_node(size, flags, node, caller); } EXPORT_SYMBOL(__kmalloc_node_track_caller); #else void *__kmalloc_node(size_t size, gfp_t flags, int node) { - return __do_kmalloc_node(size, flags, node, NULL); + return __do_kmalloc_node(size, flags, node, 0); } EXPORT_SYMBOL(__kmalloc_node); #endif /* CONFIG_DEBUG_SLAB || CONFIG_TRACING */ @@ -3921,7 +3919,7 @@ EXPORT_SYMBOL(__kmalloc_node); * @caller: function caller for debug tracking of the caller */ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags, - void *caller) + unsigned long caller) { struct kmem_cache *cachep; void *ret; @@ -3936,7 +3934,7 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags, return cachep; ret = __cache_alloc(cachep, flags, caller); - trace_kmalloc((unsigned long) caller, ret, + trace_kmalloc(caller, ret, size, cachep->size, flags); return ret; @@ -3946,20 +3944,20 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags, #if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_TRACING) void *__kmalloc(size_t size, gfp_t flags) { - return __do_kmalloc(size, flags, __builtin_return_address(0)); + return __do_kmalloc(size, flags, _RET_IP_); } EXPORT_SYMBOL(__kmalloc); void *__kmalloc_track_caller(size_t size, gfp_t flags, unsigned long caller) { - return __do_kmalloc(size, flags, (void *)caller); + return __do_kmalloc(size, flags, caller); } EXPORT_SYMBOL(__kmalloc_track_caller); #else void *__kmalloc(size_t size, gfp_t flags) { - return __do_kmalloc(size, flags, NULL); + return __do_kmalloc(size, flags, 0); } EXPORT_SYMBOL(__kmalloc); #endif @@ -3980,7 +3978,7 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp) debug_check_no_locks_freed(objp, cachep->object_size); if (!(cachep->flags & SLAB_DEBUG_OBJECTS)) debug_check_no_obj_freed(objp, cachep->object_size); - __cache_free(cachep, objp, __builtin_return_address(0)); + __cache_free(cachep, objp, _RET_IP_); local_irq_restore(flags); trace_kmem_cache_free(_RET_IP_, objp); @@ -4011,7 +4009,7 @@ void kfree(const void *objp) debug_check_no_locks_freed(objp, c->object_size); debug_check_no_obj_freed(objp, c->object_size); - __cache_free(c, (void *)objp, __builtin_return_address(0)); + __cache_free(c, (void *)objp, _RET_IP_); local_irq_restore(flags); } EXPORT_SYMBOL(kfree); From 4052147c0afa1cf05780ed846f37e87cdde9f628 Mon Sep 17 00:00:00 2001 From: Ezequiel Garcia Date: Sat, 8 Sep 2012 17:47:56 -0300 Subject: [PATCH 17/21] mm, slab: Match SLAB and SLUB kmem_cache_alloc_xxx_trace() prototype This long (seemingly unnecessary) patch does not fix anything and its only goal is to produce common code between SLAB and SLUB. Cc: Christoph Lameter Signed-off-by: Ezequiel Garcia Signed-off-by: Pekka Enberg --- include/linux/slab_def.h | 7 +++---- mm/slab.c | 10 +++++----- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/include/linux/slab_def.h b/include/linux/slab_def.h index 604ebc832a9a..e98caebdd0bc 100644 --- a/include/linux/slab_def.h +++ b/include/linux/slab_def.h @@ -111,11 +111,10 @@ void *kmem_cache_alloc(struct kmem_cache *, gfp_t); void *__kmalloc(size_t size, gfp_t flags); #ifdef CONFIG_TRACING -extern void *kmem_cache_alloc_trace(size_t size, - struct kmem_cache *cachep, gfp_t flags); +extern void *kmem_cache_alloc_trace(struct kmem_cache *, gfp_t, size_t); #else static __always_inline void * -kmem_cache_alloc_trace(size_t size, struct kmem_cache *cachep, gfp_t flags) +kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size) { return kmem_cache_alloc(cachep, flags); } @@ -148,7 +147,7 @@ found: #endif cachep = malloc_sizes[i].cs_cachep; - ret = kmem_cache_alloc_trace(size, cachep, flags); + ret = kmem_cache_alloc_trace(cachep, flags, size); return ret; } diff --git a/mm/slab.c b/mm/slab.c index 6c356793612d..4c54a2357937 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -3835,7 +3835,7 @@ EXPORT_SYMBOL(kmem_cache_alloc); #ifdef CONFIG_TRACING void * -kmem_cache_alloc_trace(size_t size, struct kmem_cache *cachep, gfp_t flags) +kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size) { void *ret; @@ -3862,10 +3862,10 @@ void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid) EXPORT_SYMBOL(kmem_cache_alloc_node); #ifdef CONFIG_TRACING -void *kmem_cache_alloc_node_trace(size_t size, - struct kmem_cache *cachep, +void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep, gfp_t flags, - int nodeid) + int nodeid, + size_t size) { void *ret; @@ -3887,7 +3887,7 @@ __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller) cachep = kmem_find_general_cachep(size, flags); if (unlikely(ZERO_OR_NULL_PTR(cachep))) return cachep; - return kmem_cache_alloc_node_trace(size, cachep, flags, node); + return kmem_cache_alloc_node_trace(cachep, flags, node, size); } #if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_TRACING) From 48356303ff8cce7036f13a23df9119d3f47461ce Mon Sep 17 00:00:00 2001 From: Ezequiel Garcia Date: Sat, 8 Sep 2012 17:47:57 -0300 Subject: [PATCH 18/21] mm, slab: Rename __cache_alloc() -> slab_alloc() This patch does not fix anything and its only goal is to produce common code between SLAB and SLUB. Cc: Christoph Lameter Signed-off-by: Ezequiel Garcia Signed-off-by: Pekka Enberg --- mm/slab.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mm/slab.c b/mm/slab.c index 4c54a2357937..d011030e9613 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -3561,7 +3561,7 @@ done: * Fallback to other node is possible if __GFP_THISNODE is not set. */ static __always_inline void * -__cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, +slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, unsigned long caller) { unsigned long save_flags; @@ -3648,7 +3648,7 @@ __do_cache_alloc(struct kmem_cache *cachep, gfp_t flags) #endif /* CONFIG_NUMA */ static __always_inline void * -__cache_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller) +slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller) { unsigned long save_flags; void *objp; @@ -3824,7 +3824,7 @@ static inline void __cache_free(struct kmem_cache *cachep, void *objp, */ void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) { - void *ret = __cache_alloc(cachep, flags, _RET_IP_); + void *ret = slab_alloc(cachep, flags, _RET_IP_); trace_kmem_cache_alloc(_RET_IP_, ret, cachep->object_size, cachep->size, flags); @@ -3839,7 +3839,7 @@ kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size) { void *ret; - ret = __cache_alloc(cachep, flags, _RET_IP_); + ret = slab_alloc(cachep, flags, _RET_IP_); trace_kmalloc(_RET_IP_, ret, size, cachep->size, flags); @@ -3851,7 +3851,7 @@ EXPORT_SYMBOL(kmem_cache_alloc_trace); #ifdef CONFIG_NUMA void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid) { - void *ret = __cache_alloc_node(cachep, flags, nodeid, _RET_IP_); + void *ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_); trace_kmem_cache_alloc_node(_RET_IP_, ret, cachep->object_size, cachep->size, @@ -3869,7 +3869,7 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep, { void *ret; - ret = __cache_alloc_node(cachep, flags, nodeid, _RET_IP); + ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP); trace_kmalloc_node(_RET_IP_, ret, size, cachep->size, @@ -3932,7 +3932,7 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags, cachep = __find_general_cachep(size, flags); if (unlikely(ZERO_OR_NULL_PTR(cachep))) return cachep; - ret = __cache_alloc(cachep, flags, caller); + ret = slab_alloc(cachep, flags, caller); trace_kmalloc(caller, ret, size, cachep->size, flags); From 2b847c3cb4f8565911bd8ce59b8428e635c90594 Mon Sep 17 00:00:00 2001 From: Ezequiel Garcia Date: Sat, 8 Sep 2012 17:47:58 -0300 Subject: [PATCH 19/21] mm, slub: Rename slab_alloc() -> slab_alloc_node() to match SLAB This patch does not fix anything, and its only goal is to enable us to obtain some common code between SLAB and SLUB. Neither behavior nor produced code is affected. Cc: Christoph Lameter Signed-off-by: Ezequiel Garcia Signed-off-by: Pekka Enberg --- mm/slub.c | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index a6d043e13266..f074f756405a 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2313,7 +2313,7 @@ new_slab: * * Otherwise we can simply pick the next object from the lockless free list. */ -static __always_inline void *slab_alloc(struct kmem_cache *s, +static __always_inline void *slab_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node, unsigned long addr) { void **object; @@ -2383,9 +2383,15 @@ redo: return object; } +static __always_inline void *slab_alloc(struct kmem_cache *s, + gfp_t gfpflags, unsigned long addr) +{ + return slab_alloc_node(s, gfpflags, NUMA_NO_NODE, addr); +} + void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags) { - void *ret = slab_alloc(s, gfpflags, NUMA_NO_NODE, _RET_IP_); + void *ret = slab_alloc(s, gfpflags, _RET_IP_); trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size, s->size, gfpflags); @@ -2396,7 +2402,7 @@ EXPORT_SYMBOL(kmem_cache_alloc); #ifdef CONFIG_TRACING void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size) { - void *ret = slab_alloc(s, gfpflags, NUMA_NO_NODE, _RET_IP_); + void *ret = slab_alloc(s, gfpflags, _RET_IP_); trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags); return ret; } @@ -2414,7 +2420,7 @@ EXPORT_SYMBOL(kmalloc_order_trace); #ifdef CONFIG_NUMA void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node) { - void *ret = slab_alloc(s, gfpflags, node, _RET_IP_); + void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_); trace_kmem_cache_alloc_node(_RET_IP_, ret, s->object_size, s->size, gfpflags, node); @@ -2428,7 +2434,7 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *s, gfp_t gfpflags, int node, size_t size) { - void *ret = slab_alloc(s, gfpflags, node, _RET_IP_); + void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_); trace_kmalloc_node(_RET_IP_, ret, size, s->size, gfpflags, node); @@ -3366,7 +3372,7 @@ void *__kmalloc(size_t size, gfp_t flags) if (unlikely(ZERO_OR_NULL_PTR(s))) return s; - ret = slab_alloc(s, flags, NUMA_NO_NODE, _RET_IP_); + ret = slab_alloc(s, flags, _RET_IP_); trace_kmalloc(_RET_IP_, ret, size, s->size, flags); @@ -3409,7 +3415,7 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node) if (unlikely(ZERO_OR_NULL_PTR(s))) return s; - ret = slab_alloc(s, flags, node, _RET_IP_); + ret = slab_alloc_node(s, flags, node, _RET_IP_); trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node); @@ -4037,7 +4043,7 @@ void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller) if (unlikely(ZERO_OR_NULL_PTR(s))) return s; - ret = slab_alloc(s, gfpflags, NUMA_NO_NODE, caller); + ret = slab_alloc(s, gfpflags, caller); /* Honor the call site pointer we received. */ trace_kmalloc(caller, ret, size, s->size, gfpflags); @@ -4067,7 +4073,7 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags, if (unlikely(ZERO_OR_NULL_PTR(s))) return s; - ret = slab_alloc(s, gfpflags, node, caller); + ret = slab_alloc_node(s, gfpflags, node, caller); /* Honor the call site pointer we received. */ trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node); From 788e1aadadd0d5a9cbffce10c34840b4072bc733 Mon Sep 17 00:00:00 2001 From: Fengguang Wu Date: Fri, 28 Sep 2012 16:34:05 +0800 Subject: [PATCH 20/21] slub: init_kmem_cache_cpus() and put_cpu_partial() can be static Acked-by: Glauber Costa Acked-by: Christoph Lameter Acked-by: David Rientjes Signed-off-by: Fengguang Wu Signed-off-by: Pekka Enberg --- mm/slub.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index f074f756405a..944b4edaeb18 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1716,7 +1716,7 @@ static inline void note_cmpxchg_failure(const char *n, stat(s, CMPXCHG_DOUBLE_CPU_FAIL); } -void init_kmem_cache_cpus(struct kmem_cache *s) +static void init_kmem_cache_cpus(struct kmem_cache *s) { int cpu; @@ -1941,7 +1941,7 @@ static void unfreeze_partials(struct kmem_cache *s) * If we did not find a slot then simply move all the partials to the * per node partial list. */ -int put_cpu_partial(struct kmem_cache *s, struct page *page, int drain) +static int put_cpu_partial(struct kmem_cache *s, struct page *page, int drain) { struct page *oldpage; int pages; From 608da7e3fc7259eca0d983b31bc8915af14cf15e Mon Sep 17 00:00:00 2001 From: Tetsuo Handa Date: Sun, 30 Sep 2012 17:28:25 +0900 Subject: [PATCH 21/21] slab: Fix build failure in __kmem_cache_create() Fix build failure with CONFIG_DEBUG_SLAB=y && CONFIG_DEBUG_PAGEALLOC=y caused by commit 8a13a4cc "mm/sl[aou]b: Shrink __kmem_cache_create() parameter lists". mm/slab.c: In function '__kmem_cache_create': mm/slab.c:2474: error: 'align' undeclared (first use in this function) mm/slab.c:2474: error: (Each undeclared identifier is reported only once mm/slab.c:2474: error: for each function it appears in.) make[1]: *** [mm/slab.o] Error 1 make: *** [mm] Error 2 Acked-by: Christoph Lameter Signed-off-by: Tetsuo Handa Signed-off-by: Pekka Enberg --- mm/slab.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mm/slab.c b/mm/slab.c index d011030e9613..c4f6e14c2c71 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -2492,8 +2492,9 @@ __kmem_cache_create (const char *name, size_t size, size_t align, } #if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC) if (size >= malloc_sizes[INDEX_L3 + 1].cs_size - && cachep->object_size > cache_line_size() && ALIGN(size, align) < PAGE_SIZE) { - cachep->obj_offset += PAGE_SIZE - ALIGN(size, align); + && cachep->object_size > cache_line_size() + && ALIGN(size, cachep->align) < PAGE_SIZE) { + cachep->obj_offset += PAGE_SIZE - ALIGN(size, cachep->align); size = PAGE_SIZE; } #endif