mm: Enable SLUB for RT
Make SLUB RT aware and remove the restriction in Kconfig. Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
This commit is contained in:
Thomas Gleixner
parent
d0392f7786
commit
0c6685b11f
|
|
@ -273,7 +273,11 @@ static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)
|
|||
* The slab lists for all objects.
|
||||
*/
|
||||
struct kmem_cache_node {
|
||||
#ifdef CONFIG_SLUB
|
||||
raw_spinlock_t list_lock;
|
||||
#else
|
||||
spinlock_t list_lock;
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_SLAB
|
||||
struct list_head slabs_partial; /* partial list first, better asm code */
|
||||
|
|
|
|||
119
mm/slub.c
119
mm/slub.c
|
|
@ -1109,7 +1109,7 @@ static noinline struct kmem_cache_node *free_debug_processing(
|
|||
{
|
||||
struct kmem_cache_node *n = get_node(s, page_to_nid(page));
|
||||
|
||||
spin_lock_irqsave(&n->list_lock, *flags);
|
||||
raw_spin_lock_irqsave(&n->list_lock, *flags);
|
||||
slab_lock(page);
|
||||
|
||||
if (!check_slab(s, page))
|
||||
|
|
@ -1157,7 +1157,7 @@ out:
|
|||
|
||||
fail:
|
||||
slab_unlock(page);
|
||||
spin_unlock_irqrestore(&n->list_lock, *flags);
|
||||
raw_spin_unlock_irqrestore(&n->list_lock, *flags);
|
||||
slab_fix(s, "Object at 0x%p not freed", object);
|
||||
return NULL;
|
||||
}
|
||||
|
|
@ -1310,6 +1310,12 @@ static inline void slab_free_hook(struct kmem_cache *s, void *x)
|
|||
|
||||
#endif /* CONFIG_SLUB_DEBUG */
|
||||
|
||||
struct slub_free_list {
|
||||
raw_spinlock_t lock;
|
||||
struct list_head list;
|
||||
};
|
||||
static DEFINE_PER_CPU(struct slub_free_list, slub_free_list);
|
||||
|
||||
/*
|
||||
* Slab allocation and freeing
|
||||
*/
|
||||
|
|
@ -1334,7 +1340,11 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
|
|||
|
||||
flags &= gfp_allowed_mask;
|
||||
|
||||
#ifdef CONFIG_PREEMPT_RT_FULL
|
||||
if (system_state == SYSTEM_RUNNING)
|
||||
#else
|
||||
if (flags & __GFP_WAIT)
|
||||
#endif
|
||||
local_irq_enable();
|
||||
|
||||
flags |= s->allocflags;
|
||||
|
|
@ -1374,7 +1384,11 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
|
|||
kmemcheck_mark_unallocated_pages(page, pages);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_PREEMPT_RT_FULL
|
||||
if (system_state == SYSTEM_RUNNING)
|
||||
#else
|
||||
if (flags & __GFP_WAIT)
|
||||
#endif
|
||||
local_irq_disable();
|
||||
if (!page)
|
||||
return NULL;
|
||||
|
|
@ -1471,6 +1485,16 @@ static void __free_slab(struct kmem_cache *s, struct page *page)
|
|||
__free_memcg_kmem_pages(page, order);
|
||||
}
|
||||
|
||||
static void free_delayed(struct list_head *h)
|
||||
{
|
||||
while(!list_empty(h)) {
|
||||
struct page *page = list_first_entry(h, struct page, lru);
|
||||
|
||||
list_del(&page->lru);
|
||||
__free_slab(page->slab_cache, page);
|
||||
}
|
||||
}
|
||||
|
||||
#define need_reserve_slab_rcu \
|
||||
(sizeof(((struct page *)NULL)->lru) < sizeof(struct rcu_head))
|
||||
|
||||
|
|
@ -1505,6 +1529,12 @@ static void free_slab(struct kmem_cache *s, struct page *page)
|
|||
}
|
||||
|
||||
call_rcu(head, rcu_free_slab);
|
||||
} else if (irqs_disabled()) {
|
||||
struct slub_free_list *f = &__get_cpu_var(slub_free_list);
|
||||
|
||||
raw_spin_lock(&f->lock);
|
||||
list_add(&page->lru, &f->list);
|
||||
raw_spin_unlock(&f->lock);
|
||||
} else
|
||||
__free_slab(s, page);
|
||||
}
|
||||
|
|
@ -1618,7 +1648,7 @@ static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,
|
|||
if (!n || !n->nr_partial)
|
||||
return NULL;
|
||||
|
||||
spin_lock(&n->list_lock);
|
||||
raw_spin_lock(&n->list_lock);
|
||||
list_for_each_entry_safe(page, page2, &n->partial, lru) {
|
||||
void *t;
|
||||
|
||||
|
|
@ -1643,7 +1673,7 @@ static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,
|
|||
break;
|
||||
|
||||
}
|
||||
spin_unlock(&n->list_lock);
|
||||
raw_spin_unlock(&n->list_lock);
|
||||
return object;
|
||||
}
|
||||
|
||||
|
|
@ -1884,7 +1914,7 @@ redo:
|
|||
* that acquire_slab() will see a slab page that
|
||||
* is frozen
|
||||
*/
|
||||
spin_lock(&n->list_lock);
|
||||
raw_spin_lock(&n->list_lock);
|
||||
}
|
||||
} else {
|
||||
m = M_FULL;
|
||||
|
|
@ -1895,7 +1925,7 @@ redo:
|
|||
* slabs from diagnostic functions will not see
|
||||
* any frozen slabs.
|
||||
*/
|
||||
spin_lock(&n->list_lock);
|
||||
raw_spin_lock(&n->list_lock);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1930,7 +1960,7 @@ redo:
|
|||
goto redo;
|
||||
|
||||
if (lock)
|
||||
spin_unlock(&n->list_lock);
|
||||
raw_spin_unlock(&n->list_lock);
|
||||
|
||||
if (m == M_FREE) {
|
||||
stat(s, DEACTIVATE_EMPTY);
|
||||
|
|
@ -1962,10 +1992,10 @@ static void unfreeze_partials(struct kmem_cache *s,
|
|||
n2 = get_node(s, page_to_nid(page));
|
||||
if (n != n2) {
|
||||
if (n)
|
||||
spin_unlock(&n->list_lock);
|
||||
raw_spin_unlock(&n->list_lock);
|
||||
|
||||
n = n2;
|
||||
spin_lock(&n->list_lock);
|
||||
raw_spin_lock(&n->list_lock);
|
||||
}
|
||||
|
||||
do {
|
||||
|
|
@ -1994,7 +2024,7 @@ static void unfreeze_partials(struct kmem_cache *s,
|
|||
}
|
||||
|
||||
if (n)
|
||||
spin_unlock(&n->list_lock);
|
||||
raw_spin_unlock(&n->list_lock);
|
||||
|
||||
while (discard_page) {
|
||||
page = discard_page;
|
||||
|
|
@ -2032,14 +2062,21 @@ static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
|
|||
pobjects = oldpage->pobjects;
|
||||
pages = oldpage->pages;
|
||||
if (drain && pobjects > s->cpu_partial) {
|
||||
struct slub_free_list *f;
|
||||
unsigned long flags;
|
||||
LIST_HEAD(tofree);
|
||||
/*
|
||||
* partial array is full. Move the existing
|
||||
* set to the per node partial list.
|
||||
*/
|
||||
local_irq_save(flags);
|
||||
unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
|
||||
f = &__get_cpu_var(slub_free_list);
|
||||
raw_spin_lock(&f->lock);
|
||||
list_splice_init(&f->list, &tofree);
|
||||
raw_spin_unlock(&f->lock);
|
||||
local_irq_restore(flags);
|
||||
free_delayed(&tofree);
|
||||
oldpage = NULL;
|
||||
pobjects = 0;
|
||||
pages = 0;
|
||||
|
|
@ -2103,7 +2140,22 @@ static bool has_cpu_slab(int cpu, void *info)
|
|||
|
||||
static void flush_all(struct kmem_cache *s)
|
||||
{
|
||||
LIST_HEAD(tofree);
|
||||
int cpu;
|
||||
|
||||
on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1, GFP_ATOMIC);
|
||||
for_each_online_cpu(cpu) {
|
||||
struct slub_free_list *f;
|
||||
|
||||
if (!has_cpu_slab(cpu, s))
|
||||
continue;
|
||||
|
||||
f = &per_cpu(slub_free_list, cpu);
|
||||
raw_spin_lock_irq(&f->lock);
|
||||
list_splice_init(&f->list, &tofree);
|
||||
raw_spin_unlock_irq(&f->lock);
|
||||
free_delayed(&tofree);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -2131,10 +2183,10 @@ static unsigned long count_partial(struct kmem_cache_node *n,
|
|||
unsigned long x = 0;
|
||||
struct page *page;
|
||||
|
||||
spin_lock_irqsave(&n->list_lock, flags);
|
||||
raw_spin_lock_irqsave(&n->list_lock, flags);
|
||||
list_for_each_entry(page, &n->partial, lru)
|
||||
x += get_count(page);
|
||||
spin_unlock_irqrestore(&n->list_lock, flags);
|
||||
raw_spin_unlock_irqrestore(&n->list_lock, flags);
|
||||
return x;
|
||||
}
|
||||
|
||||
|
|
@ -2277,9 +2329,11 @@ static inline void *get_freelist(struct kmem_cache *s, struct page *page)
|
|||
static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
|
||||
unsigned long addr, struct kmem_cache_cpu *c)
|
||||
{
|
||||
struct slub_free_list *f;
|
||||
void *freelist;
|
||||
struct page *page;
|
||||
unsigned long flags;
|
||||
LIST_HEAD(tofree);
|
||||
|
||||
local_irq_save(flags);
|
||||
#ifdef CONFIG_PREEMPT
|
||||
|
|
@ -2342,7 +2396,13 @@ load_freelist:
|
|||
VM_BUG_ON(!c->page->frozen);
|
||||
c->freelist = get_freepointer(s, freelist);
|
||||
c->tid = next_tid(c->tid);
|
||||
out:
|
||||
f = &__get_cpu_var(slub_free_list);
|
||||
raw_spin_lock(&f->lock);
|
||||
list_splice_init(&f->list, &tofree);
|
||||
raw_spin_unlock(&f->lock);
|
||||
local_irq_restore(flags);
|
||||
free_delayed(&tofree);
|
||||
return freelist;
|
||||
|
||||
new_slab:
|
||||
|
|
@ -2360,9 +2420,7 @@ new_slab:
|
|||
if (unlikely(!freelist)) {
|
||||
if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit())
|
||||
slab_out_of_memory(s, gfpflags, node);
|
||||
|
||||
local_irq_restore(flags);
|
||||
return NULL;
|
||||
goto out;
|
||||
}
|
||||
|
||||
page = c->page;
|
||||
|
|
@ -2377,8 +2435,7 @@ new_slab:
|
|||
deactivate_slab(s, page, get_freepointer(s, freelist));
|
||||
c->page = NULL;
|
||||
c->freelist = NULL;
|
||||
local_irq_restore(flags);
|
||||
return freelist;
|
||||
goto out;
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -2550,7 +2607,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
|
|||
|
||||
do {
|
||||
if (unlikely(n)) {
|
||||
spin_unlock_irqrestore(&n->list_lock, flags);
|
||||
raw_spin_unlock_irqrestore(&n->list_lock, flags);
|
||||
n = NULL;
|
||||
}
|
||||
prior = page->freelist;
|
||||
|
|
@ -2582,7 +2639,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
|
|||
* Otherwise the list_lock will synchronize with
|
||||
* other processors updating the list of slabs.
|
||||
*/
|
||||
spin_lock_irqsave(&n->list_lock, flags);
|
||||
raw_spin_lock_irqsave(&n->list_lock, flags);
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -2624,7 +2681,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
|
|||
add_partial(n, page, DEACTIVATE_TO_TAIL);
|
||||
stat(s, FREE_ADD_PARTIAL);
|
||||
}
|
||||
spin_unlock_irqrestore(&n->list_lock, flags);
|
||||
raw_spin_unlock_irqrestore(&n->list_lock, flags);
|
||||
return;
|
||||
|
||||
slab_empty:
|
||||
|
|
@ -2639,7 +2696,7 @@ slab_empty:
|
|||
remove_full(s, n, page);
|
||||
}
|
||||
|
||||
spin_unlock_irqrestore(&n->list_lock, flags);
|
||||
raw_spin_unlock_irqrestore(&n->list_lock, flags);
|
||||
stat(s, FREE_SLAB);
|
||||
discard_slab(s, page);
|
||||
}
|
||||
|
|
@ -2841,7 +2898,7 @@ static void
|
|||
init_kmem_cache_node(struct kmem_cache_node *n)
|
||||
{
|
||||
n->nr_partial = 0;
|
||||
spin_lock_init(&n->list_lock);
|
||||
raw_spin_lock_init(&n->list_lock);
|
||||
INIT_LIST_HEAD(&n->partial);
|
||||
#ifdef CONFIG_SLUB_DEBUG
|
||||
atomic_long_set(&n->nr_slabs, 0);
|
||||
|
|
@ -3431,7 +3488,7 @@ int kmem_cache_shrink(struct kmem_cache *s)
|
|||
for (i = 0; i < objects; i++)
|
||||
INIT_LIST_HEAD(slabs_by_inuse + i);
|
||||
|
||||
spin_lock_irqsave(&n->list_lock, flags);
|
||||
raw_spin_lock_irqsave(&n->list_lock, flags);
|
||||
|
||||
/*
|
||||
* Build lists indexed by the items in use in each slab.
|
||||
|
|
@ -3452,7 +3509,7 @@ int kmem_cache_shrink(struct kmem_cache *s)
|
|||
for (i = objects - 1; i > 0; i--)
|
||||
list_splice(slabs_by_inuse + i, n->partial.prev);
|
||||
|
||||
spin_unlock_irqrestore(&n->list_lock, flags);
|
||||
raw_spin_unlock_irqrestore(&n->list_lock, flags);
|
||||
|
||||
/* Release empty slabs */
|
||||
list_for_each_entry_safe(page, t, slabs_by_inuse, lru)
|
||||
|
|
@ -3628,6 +3685,12 @@ void __init kmem_cache_init(void)
|
|||
{
|
||||
static __initdata struct kmem_cache boot_kmem_cache,
|
||||
boot_kmem_cache_node;
|
||||
int cpu;
|
||||
|
||||
for_each_possible_cpu(cpu) {
|
||||
raw_spin_lock_init(&per_cpu(slub_free_list, cpu).lock);
|
||||
INIT_LIST_HEAD(&per_cpu(slub_free_list, cpu).list);
|
||||
}
|
||||
|
||||
if (debug_guardpage_minorder())
|
||||
slub_max_order = 0;
|
||||
|
|
@ -3932,7 +3995,7 @@ static int validate_slab_node(struct kmem_cache *s,
|
|||
struct page *page;
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&n->list_lock, flags);
|
||||
raw_spin_lock_irqsave(&n->list_lock, flags);
|
||||
|
||||
list_for_each_entry(page, &n->partial, lru) {
|
||||
validate_slab_slab(s, page, map);
|
||||
|
|
@ -3955,7 +4018,7 @@ static int validate_slab_node(struct kmem_cache *s,
|
|||
atomic_long_read(&n->nr_slabs));
|
||||
|
||||
out:
|
||||
spin_unlock_irqrestore(&n->list_lock, flags);
|
||||
raw_spin_unlock_irqrestore(&n->list_lock, flags);
|
||||
return count;
|
||||
}
|
||||
|
||||
|
|
@ -4145,12 +4208,12 @@ static int list_locations(struct kmem_cache *s, char *buf,
|
|||
if (!atomic_long_read(&n->nr_slabs))
|
||||
continue;
|
||||
|
||||
spin_lock_irqsave(&n->list_lock, flags);
|
||||
raw_spin_lock_irqsave(&n->list_lock, flags);
|
||||
list_for_each_entry(page, &n->partial, lru)
|
||||
process_slab(&t, s, page, alloc, map);
|
||||
list_for_each_entry(page, &n->full, lru)
|
||||
process_slab(&t, s, page, alloc, map);
|
||||
spin_unlock_irqrestore(&n->list_lock, flags);
|
||||
raw_spin_unlock_irqrestore(&n->list_lock, flags);
|
||||
}
|
||||
|
||||
for (i = 0; i < t.count; i++) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue