SLUB: Avoid touching page struct when freeing to per cpu slab
Set c->node to -1 if we allocate from a debug slab instead for SlabDebug which requires access the page struct cacheline. Signed-off-by: Christoph Lameter <clameter@sgi.com> Tested-by: Alexey Dobriyan <adobriyan@sw.ru> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
parent
b3fba8da65
commit
ee3c72a14b
14
mm/slub.c
14
mm/slub.c
|
@ -1537,6 +1537,7 @@ debug:
|
||||||
|
|
||||||
c->page->inuse++;
|
c->page->inuse++;
|
||||||
c->page->freelist = object[c->offset];
|
c->page->freelist = object[c->offset];
|
||||||
|
c->node = -1;
|
||||||
slab_unlock(c->page);
|
slab_unlock(c->page);
|
||||||
return object;
|
return object;
|
||||||
}
|
}
|
||||||
|
@ -1560,8 +1561,7 @@ static void __always_inline *slab_alloc(struct kmem_cache *s,
|
||||||
|
|
||||||
local_irq_save(flags);
|
local_irq_save(flags);
|
||||||
c = get_cpu_slab(s, smp_processor_id());
|
c = get_cpu_slab(s, smp_processor_id());
|
||||||
if (unlikely(!c->page || !c->freelist ||
|
if (unlikely(!c->freelist || !node_match(c, node)))
|
||||||
!node_match(c, node)))
|
|
||||||
|
|
||||||
object = __slab_alloc(s, gfpflags, node, addr, c);
|
object = __slab_alloc(s, gfpflags, node, addr, c);
|
||||||
|
|
||||||
|
@ -1670,7 +1670,7 @@ static void __always_inline slab_free(struct kmem_cache *s,
|
||||||
local_irq_save(flags);
|
local_irq_save(flags);
|
||||||
debug_check_no_locks_freed(object, s->objsize);
|
debug_check_no_locks_freed(object, s->objsize);
|
||||||
c = get_cpu_slab(s, smp_processor_id());
|
c = get_cpu_slab(s, smp_processor_id());
|
||||||
if (likely(page == c->page && !SlabDebug(page))) {
|
if (likely(page == c->page && c->node >= 0)) {
|
||||||
object[c->offset] = c->freelist;
|
object[c->offset] = c->freelist;
|
||||||
c->freelist = object;
|
c->freelist = object;
|
||||||
} else
|
} else
|
||||||
|
@ -3250,12 +3250,16 @@ static unsigned long slab_objects(struct kmem_cache *s,
|
||||||
|
|
||||||
for_each_possible_cpu(cpu) {
|
for_each_possible_cpu(cpu) {
|
||||||
struct page *page;
|
struct page *page;
|
||||||
|
int node;
|
||||||
struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
|
struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
|
||||||
|
|
||||||
if (!c)
|
if (!c)
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
page = c->page;
|
page = c->page;
|
||||||
|
node = c->node;
|
||||||
|
if (node < 0)
|
||||||
|
continue;
|
||||||
if (page) {
|
if (page) {
|
||||||
if (flags & SO_CPU) {
|
if (flags & SO_CPU) {
|
||||||
int x = 0;
|
int x = 0;
|
||||||
|
@ -3265,9 +3269,9 @@ static unsigned long slab_objects(struct kmem_cache *s,
|
||||||
else
|
else
|
||||||
x = 1;
|
x = 1;
|
||||||
total += x;
|
total += x;
|
||||||
nodes[c->node] += x;
|
nodes[node] += x;
|
||||||
}
|
}
|
||||||
per_cpu[c->node]++;
|
per_cpu[node]++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue