slab: reorganize memcg_cache_params

We're going to change how memcg caches are iterated.  In preparation,
clean up and reorganize memcg_cache_params.

* The shared ->list is replaced by ->children in root and
  ->children_node in children.

* ->is_root_cache is removed.  Instead ->root_cache is moved out of
  the child union and now used by both root and children.  NULL
  indicates root cache.  Non-NULL a memcg one.

This patch doesn't cause any observable behavior changes.

Link: http://lkml.kernel.org/r/20170117235411.9408-5-tj@kernel.org
Signed-off-by: Tejun Heo <tj@kernel.org>
Acked-by: Vladimir Davydov <vdavydov.dev@gmail.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
Tejun Heo 2017-02-22 15:41:17 -08:00 committed by Linus Torvalds
parent 657dc2f972
commit 9eeadc8b6e
3 changed files with 40 additions and 24 deletions

View File

@ -545,22 +545,37 @@ struct memcg_cache_array {
* array to be accessed without taking any locks, on relocation we free the old * array to be accessed without taking any locks, on relocation we free the old
* version only after a grace period. * version only after a grace period.
* *
* Child caches will hold extra metadata needed for its operation. Fields are: * Root and child caches hold different metadata.
* *
* @memcg: pointer to the memcg this cache belongs to * @root_cache: Common to root and child caches. NULL for root, pointer to
* @root_cache: pointer to the global, root cache, this cache was derived from * the root cache for children.
* *
* Both root and child caches of the same kind are linked into a list chained * The following fields are specific to root caches.
* through @list. *
* @memcg_caches: kmemcg ID indexed table of child caches. This table is
* used to index child cachces during allocation and cleared
* early during shutdown.
*
* @children: List of all child caches. While the child caches are also
* reachable through @memcg_caches, a child cache remains on
* this list until it is actually destroyed.
*
* The following fields are specific to child caches.
*
* @memcg: Pointer to the memcg this cache belongs to.
*
* @children_node: List node for @root_cache->children list.
*/ */
struct memcg_cache_params { struct memcg_cache_params {
bool is_root_cache; struct kmem_cache *root_cache;
struct list_head list;
union { union {
struct memcg_cache_array __rcu *memcg_caches; struct {
struct memcg_cache_array __rcu *memcg_caches;
struct list_head children;
};
struct { struct {
struct mem_cgroup *memcg; struct mem_cgroup *memcg;
struct kmem_cache *root_cache; struct list_head children_node;
}; };
}; };
}; };

View File

@ -206,12 +206,12 @@ int __kmem_cache_alloc_bulk(struct kmem_cache *, gfp_t, size_t, void **);
* slab_mutex. * slab_mutex.
*/ */
#define for_each_memcg_cache(iter, root) \ #define for_each_memcg_cache(iter, root) \
list_for_each_entry(iter, &(root)->memcg_params.list, \ list_for_each_entry(iter, &(root)->memcg_params.children, \
memcg_params.list) memcg_params.children_node)
static inline bool is_root_cache(struct kmem_cache *s) static inline bool is_root_cache(struct kmem_cache *s)
{ {
return s->memcg_params.is_root_cache; return !s->memcg_params.root_cache;
} }
static inline bool slab_equal_or_root(struct kmem_cache *s, static inline bool slab_equal_or_root(struct kmem_cache *s,

View File

@ -140,9 +140,9 @@ int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t nr,
#if defined(CONFIG_MEMCG) && !defined(CONFIG_SLOB) #if defined(CONFIG_MEMCG) && !defined(CONFIG_SLOB)
void slab_init_memcg_params(struct kmem_cache *s) void slab_init_memcg_params(struct kmem_cache *s)
{ {
s->memcg_params.is_root_cache = true; s->memcg_params.root_cache = NULL;
INIT_LIST_HEAD(&s->memcg_params.list);
RCU_INIT_POINTER(s->memcg_params.memcg_caches, NULL); RCU_INIT_POINTER(s->memcg_params.memcg_caches, NULL);
INIT_LIST_HEAD(&s->memcg_params.children);
} }
static int init_memcg_params(struct kmem_cache *s, static int init_memcg_params(struct kmem_cache *s,
@ -150,10 +150,10 @@ static int init_memcg_params(struct kmem_cache *s,
{ {
struct memcg_cache_array *arr; struct memcg_cache_array *arr;
if (memcg) { if (root_cache) {
s->memcg_params.is_root_cache = false;
s->memcg_params.memcg = memcg;
s->memcg_params.root_cache = root_cache; s->memcg_params.root_cache = root_cache;
s->memcg_params.memcg = memcg;
INIT_LIST_HEAD(&s->memcg_params.children_node);
return 0; return 0;
} }
@ -223,7 +223,7 @@ int memcg_update_all_caches(int num_memcgs)
static void unlink_memcg_cache(struct kmem_cache *s) static void unlink_memcg_cache(struct kmem_cache *s)
{ {
list_del(&s->memcg_params.list); list_del(&s->memcg_params.children_node);
} }
#else #else
static inline int init_memcg_params(struct kmem_cache *s, static inline int init_memcg_params(struct kmem_cache *s,
@ -594,7 +594,8 @@ void memcg_create_kmem_cache(struct mem_cgroup *memcg,
goto out_unlock; goto out_unlock;
} }
list_add(&s->memcg_params.list, &root_cache->memcg_params.list); list_add(&s->memcg_params.children_node,
&root_cache->memcg_params.children);
/* /*
* Since readers won't lock (see cache_from_memcg_idx()), we need a * Since readers won't lock (see cache_from_memcg_idx()), we need a
@ -690,7 +691,7 @@ static int shutdown_memcg_caches(struct kmem_cache *s)
* list so as not to try to destroy it for a second * list so as not to try to destroy it for a second
* time while iterating over inactive caches below. * time while iterating over inactive caches below.
*/ */
list_move(&c->memcg_params.list, &busy); list_move(&c->memcg_params.children_node, &busy);
else else
/* /*
* The cache is empty and will be destroyed soon. Clear * The cache is empty and will be destroyed soon. Clear
@ -705,17 +706,17 @@ static int shutdown_memcg_caches(struct kmem_cache *s)
* Second, shutdown all caches left from memory cgroups that are now * Second, shutdown all caches left from memory cgroups that are now
* offline. * offline.
*/ */
list_for_each_entry_safe(c, c2, &s->memcg_params.list, list_for_each_entry_safe(c, c2, &s->memcg_params.children,
memcg_params.list) memcg_params.children_node)
shutdown_cache(c); shutdown_cache(c);
list_splice(&busy, &s->memcg_params.list); list_splice(&busy, &s->memcg_params.children);
/* /*
* A cache being destroyed must be empty. In particular, this means * A cache being destroyed must be empty. In particular, this means
* that all per memcg caches attached to it must be empty too. * that all per memcg caches attached to it must be empty too.
*/ */
if (!list_empty(&s->memcg_params.list)) if (!list_empty(&s->memcg_params.children))
return -EBUSY; return -EBUSY;
return 0; return 0;
} }