mm: introduce arg_lock to protect arg_start|end and env_start|end in mm_struct
mmap_sem is on the hot path of kernel, and it very contended, but it is abused too. It is used to protect arg_start|end and evn_start|end when reading /proc/$PID/cmdline and /proc/$PID/environ, but it doesn't make sense since those proc files just expect to read 4 values atomically and not related to VM, they could be set to arbitrary values by C/R. And, the mmap_sem contention may cause unexpected issue like below: INFO: task ps:14018 blocked for more than 120 seconds. Tainted: G E 4.9.79-009.ali3000.alios7.x86_64 #1 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. ps D 0 14018 1 0x00000004 Call Trace: schedule+0x36/0x80 rwsem_down_read_failed+0xf0/0x150 call_rwsem_down_read_failed+0x18/0x30 down_read+0x20/0x40 proc_pid_cmdline_read+0xd9/0x4e0 __vfs_read+0x37/0x150 vfs_read+0x96/0x130 SyS_read+0x55/0xc0 entry_SYSCALL_64_fastpath+0x1a/0xc5 Both Alexey Dobriyan and Michal Hocko suggested to use dedicated lock for them to mitigate the abuse of mmap_sem. So, introduce a new spinlock in mm_struct to protect the concurrent access to arg_start|end, env_start|end and others, as well as replace write map_sem to read to protect the race condition between prctl and sys_brk which might break check_data_rlimit(), and makes prctl more friendly to other VM operations. This patch just eliminates the abuse of mmap_sem, but it can't resolve the above hung task warning completely since the later access_remote_vm() call needs acquire mmap_sem. The mmap_sem scalability issue will be solved in the future. [yang.shi@linux.alibaba.com: add comment about mmap_sem and arg_lock] Link: http://lkml.kernel.org/r/1524077799-80690-1-git-send-email-yang.shi@linux.alibaba.com Link: http://lkml.kernel.org/r/1523730291-109696-1-git-send-email-yang.shi@linux.alibaba.com Signed-off-by: Yang Shi <yang.shi@linux.alibaba.com> Reviewed-by: Cyrill Gorcunov <gorcunov@openvz.org> Acked-by: Michal Hocko <mhocko@suse.com> Cc: Alexey Dobriyan <adobriyan@gmail.com> Cc: Matthew Wilcox <willy@infradead.org> Cc: Mateusz Guzik <mguzik@redhat.com> Cc: Kirill Tkhai <ktkhai@virtuozzo.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
parent
05fec35ebb
commit
88aa7cc688
|
@ -239,12 +239,12 @@ static ssize_t proc_pid_cmdline_read(struct file *file, char __user *buf,
|
||||||
goto out_mmput;
|
goto out_mmput;
|
||||||
}
|
}
|
||||||
|
|
||||||
down_read(&mm->mmap_sem);
|
spin_lock(&mm->arg_lock);
|
||||||
arg_start = mm->arg_start;
|
arg_start = mm->arg_start;
|
||||||
arg_end = mm->arg_end;
|
arg_end = mm->arg_end;
|
||||||
env_start = mm->env_start;
|
env_start = mm->env_start;
|
||||||
env_end = mm->env_end;
|
env_end = mm->env_end;
|
||||||
up_read(&mm->mmap_sem);
|
spin_unlock(&mm->arg_lock);
|
||||||
|
|
||||||
BUG_ON(arg_start > arg_end);
|
BUG_ON(arg_start > arg_end);
|
||||||
BUG_ON(env_start > env_end);
|
BUG_ON(env_start > env_end);
|
||||||
|
@ -927,10 +927,10 @@ static ssize_t environ_read(struct file *file, char __user *buf,
|
||||||
if (!mmget_not_zero(mm))
|
if (!mmget_not_zero(mm))
|
||||||
goto free;
|
goto free;
|
||||||
|
|
||||||
down_read(&mm->mmap_sem);
|
spin_lock(&mm->arg_lock);
|
||||||
env_start = mm->env_start;
|
env_start = mm->env_start;
|
||||||
env_end = mm->env_end;
|
env_end = mm->env_end;
|
||||||
up_read(&mm->mmap_sem);
|
spin_unlock(&mm->arg_lock);
|
||||||
|
|
||||||
while (count > 0) {
|
while (count > 0) {
|
||||||
size_t this_len, max_len;
|
size_t this_len, max_len;
|
||||||
|
|
|
@ -413,6 +413,8 @@ struct mm_struct {
|
||||||
unsigned long exec_vm; /* VM_EXEC & ~VM_WRITE & ~VM_STACK */
|
unsigned long exec_vm; /* VM_EXEC & ~VM_WRITE & ~VM_STACK */
|
||||||
unsigned long stack_vm; /* VM_STACK */
|
unsigned long stack_vm; /* VM_STACK */
|
||||||
unsigned long def_flags;
|
unsigned long def_flags;
|
||||||
|
|
||||||
|
spinlock_t arg_lock; /* protect the below fields */
|
||||||
unsigned long start_code, end_code, start_data, end_data;
|
unsigned long start_code, end_code, start_data, end_data;
|
||||||
unsigned long start_brk, brk, start_stack;
|
unsigned long start_brk, brk, start_stack;
|
||||||
unsigned long arg_start, arg_end, env_start, env_end;
|
unsigned long arg_start, arg_end, env_start, env_end;
|
||||||
|
|
|
@ -899,6 +899,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
|
||||||
mm->pinned_vm = 0;
|
mm->pinned_vm = 0;
|
||||||
memset(&mm->rss_stat, 0, sizeof(mm->rss_stat));
|
memset(&mm->rss_stat, 0, sizeof(mm->rss_stat));
|
||||||
spin_lock_init(&mm->page_table_lock);
|
spin_lock_init(&mm->page_table_lock);
|
||||||
|
spin_lock_init(&mm->arg_lock);
|
||||||
mm_init_cpumask(mm);
|
mm_init_cpumask(mm);
|
||||||
mm_init_aio(mm);
|
mm_init_aio(mm);
|
||||||
mm_init_owner(mm, p);
|
mm_init_owner(mm, p);
|
||||||
|
|
10
kernel/sys.c
10
kernel/sys.c
|
@ -2018,7 +2018,11 @@ static int prctl_set_mm_map(int opt, const void __user *addr, unsigned long data
|
||||||
return error;
|
return error;
|
||||||
}
|
}
|
||||||
|
|
||||||
down_write(&mm->mmap_sem);
|
/*
|
||||||
|
* arg_lock protects concurent updates but we still need mmap_sem for
|
||||||
|
* read to exclude races with sys_brk.
|
||||||
|
*/
|
||||||
|
down_read(&mm->mmap_sem);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* We don't validate if these members are pointing to
|
* We don't validate if these members are pointing to
|
||||||
|
@ -2032,6 +2036,7 @@ static int prctl_set_mm_map(int opt, const void __user *addr, unsigned long data
|
||||||
* to any problem in kernel itself
|
* to any problem in kernel itself
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
spin_lock(&mm->arg_lock);
|
||||||
mm->start_code = prctl_map.start_code;
|
mm->start_code = prctl_map.start_code;
|
||||||
mm->end_code = prctl_map.end_code;
|
mm->end_code = prctl_map.end_code;
|
||||||
mm->start_data = prctl_map.start_data;
|
mm->start_data = prctl_map.start_data;
|
||||||
|
@ -2043,6 +2048,7 @@ static int prctl_set_mm_map(int opt, const void __user *addr, unsigned long data
|
||||||
mm->arg_end = prctl_map.arg_end;
|
mm->arg_end = prctl_map.arg_end;
|
||||||
mm->env_start = prctl_map.env_start;
|
mm->env_start = prctl_map.env_start;
|
||||||
mm->env_end = prctl_map.env_end;
|
mm->env_end = prctl_map.env_end;
|
||||||
|
spin_unlock(&mm->arg_lock);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Note this update of @saved_auxv is lockless thus
|
* Note this update of @saved_auxv is lockless thus
|
||||||
|
@ -2055,7 +2061,7 @@ static int prctl_set_mm_map(int opt, const void __user *addr, unsigned long data
|
||||||
if (prctl_map.auxv_size)
|
if (prctl_map.auxv_size)
|
||||||
memcpy(mm->saved_auxv, user_auxv, sizeof(user_auxv));
|
memcpy(mm->saved_auxv, user_auxv, sizeof(user_auxv));
|
||||||
|
|
||||||
up_write(&mm->mmap_sem);
|
up_read(&mm->mmap_sem);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
#endif /* CONFIG_CHECKPOINT_RESTORE */
|
#endif /* CONFIG_CHECKPOINT_RESTORE */
|
||||||
|
|
|
@ -22,6 +22,7 @@ struct mm_struct init_mm = {
|
||||||
.mm_count = ATOMIC_INIT(1),
|
.mm_count = ATOMIC_INIT(1),
|
||||||
.mmap_sem = __RWSEM_INITIALIZER(init_mm.mmap_sem),
|
.mmap_sem = __RWSEM_INITIALIZER(init_mm.mmap_sem),
|
||||||
.page_table_lock = __SPIN_LOCK_UNLOCKED(init_mm.page_table_lock),
|
.page_table_lock = __SPIN_LOCK_UNLOCKED(init_mm.page_table_lock),
|
||||||
|
.arg_lock = __SPIN_LOCK_UNLOCKED(init_mm.arg_lock),
|
||||||
.mmlist = LIST_HEAD_INIT(init_mm.mmlist),
|
.mmlist = LIST_HEAD_INIT(init_mm.mmlist),
|
||||||
.user_ns = &init_user_ns,
|
.user_ns = &init_user_ns,
|
||||||
INIT_MM_CONTEXT(init_mm)
|
INIT_MM_CONTEXT(init_mm)
|
||||||
|
|
Loading…
Reference in New Issue