mm: pagefault_disabled()
Wrap the test for pagefault_disabled() into a helper, this allows us to remove the need for current->pagefault_disabled on !-rt kernels. Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Link: http://lkml.kernel.org/n/tip-3yy517m8zsi9fpsf14xfaqkw@git.kernel.org
This commit is contained in:
parent
caf7963976
commit
93aee95b49
|
@ -107,7 +107,7 @@ do_page_fault(unsigned long address, unsigned long mmcsr,
|
|||
|
||||
/* If we're in an interrupt context, or have no user context,
|
||||
we must not take the fault. */
|
||||
if (!mm || in_atomic() || current->pagefault_disabled)
|
||||
if (!mm || pagefault_disabled())
|
||||
goto no_context;
|
||||
|
||||
#ifdef CONFIG_ALPHA_LARGE_VMALLOC
|
||||
|
|
|
@ -277,7 +277,7 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
|
|||
* If we're in an interrupt or have no user
|
||||
* context, we must not take the fault..
|
||||
*/
|
||||
if (in_atomic() || !mm || current->pagefault_disabled)
|
||||
if (!mm || pagefault_disabled())
|
||||
goto no_context;
|
||||
|
||||
if (user_mode(regs))
|
||||
|
|
|
@ -81,8 +81,7 @@ asmlinkage void do_page_fault(unsigned long ecr, struct pt_regs *regs)
|
|||
* If we're in an interrupt or have no user context, we must
|
||||
* not take the fault...
|
||||
*/
|
||||
if (in_atomic() || !mm || regs->sr & SYSREG_BIT(GM) ||
|
||||
current->pagefault_disabled)
|
||||
if (!mm || regs->sr & SYSREG_BIT(GM) || pagefault_disabled())
|
||||
goto no_context;
|
||||
|
||||
local_irq_enable();
|
||||
|
|
|
@ -113,7 +113,7 @@ do_page_fault(unsigned long address, struct pt_regs *regs,
|
|||
* user context, we must not take the fault.
|
||||
*/
|
||||
|
||||
if (in_atomic() || !mm || current->pagefault_disabled)
|
||||
if (!mm || pagefault_disabled())
|
||||
goto no_context;
|
||||
|
||||
if (user_mode(regs))
|
||||
|
|
|
@ -78,7 +78,7 @@ asmlinkage void do_page_fault(int datammu, unsigned long esr0, unsigned long ear
|
|||
* If we're in an interrupt or have no user
|
||||
* context, we must not take the fault..
|
||||
*/
|
||||
if (in_atomic() || !mm || current->pagefault_disabled)
|
||||
if (!mm || pagefault_disabled())
|
||||
goto no_context;
|
||||
|
||||
if (user_mode(__frame))
|
||||
|
|
|
@ -96,7 +96,7 @@ ia64_do_page_fault (unsigned long address, unsigned long isr, struct pt_regs *re
|
|||
/*
|
||||
* If we're in an interrupt or have no user context, we must not take the fault..
|
||||
*/
|
||||
if (in_atomic() || !mm || current->pagefault_disabled)
|
||||
if (!mm || pagefault_disabled())
|
||||
goto no_context;
|
||||
|
||||
#ifdef CONFIG_VIRTUAL_MEM_MAP
|
||||
|
|
|
@ -114,7 +114,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code,
|
|||
* If we're in an interrupt or have no user context or are running in an
|
||||
* atomic region then we must not take the fault..
|
||||
*/
|
||||
if (in_atomic() || !mm || current->pagefault_disabled
|
||||
if (!mm || pagefault_disabled())
|
||||
goto bad_area_nosemaphore;
|
||||
|
||||
if (error_code & ACE_USERMODE)
|
||||
|
|
|
@ -81,7 +81,7 @@ int do_page_fault(struct pt_regs *regs, unsigned long address,
|
|||
* If we're in an interrupt or have no user
|
||||
* context, we must not take the fault..
|
||||
*/
|
||||
if (in_atomic() || !mm || current->pagefault_disabled)
|
||||
if (!mm || pagefault_disabled())
|
||||
goto no_context;
|
||||
|
||||
if (user_mode(regs))
|
||||
|
|
|
@ -107,7 +107,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long address,
|
|||
if ((error_code & 0x13) == 0x13 || (error_code & 0x11) == 0x11)
|
||||
is_write = 0;
|
||||
|
||||
if (unlikely(in_atomic() || !mm || current->pagefault_disabled)) {
|
||||
if (unlikely(!mm || pagefault_disabled())) {
|
||||
if (kernel_mode(regs))
|
||||
goto bad_area_nosemaphore;
|
||||
|
||||
|
|
|
@ -89,7 +89,7 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write,
|
|||
* If we're in an interrupt or have no user
|
||||
* context, we must not take the fault..
|
||||
*/
|
||||
if (in_atomic() || !mm || current->pagefault_disabled)
|
||||
if (!mm || pagefault_disabled())
|
||||
goto bad_area_nosemaphore;
|
||||
|
||||
if (user_mode(regs))
|
||||
|
|
|
@ -168,7 +168,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long fault_code,
|
|||
* If we're in an interrupt or have no user
|
||||
* context, we must not take the fault..
|
||||
*/
|
||||
if (in_atomic() || !mm || current->pagefault_disabled)
|
||||
if (!mm || pagefault_disabled())
|
||||
goto no_context;
|
||||
|
||||
if ((fault_code & MMUFCR_xFC_ACCESS) == MMUFCR_xFC_ACCESS_USR)
|
||||
|
|
|
@ -207,7 +207,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long code,
|
|||
int fault;
|
||||
unsigned int flags;
|
||||
|
||||
if (in_atomic() || current->pagefault_disabled)
|
||||
if (pagefault_disabled())
|
||||
goto no_context;
|
||||
|
||||
tsk = current;
|
||||
|
|
|
@ -261,7 +261,7 @@ int __kprobes do_page_fault(struct pt_regs *regs, unsigned long address,
|
|||
if (!arch_irq_disabled_regs(regs))
|
||||
local_irq_enable();
|
||||
|
||||
if (in_atomic() || mm == NULL || current->pagefault_disabled) {
|
||||
if (in_atomic() || mm == NULL || pagefault_disabled()) {
|
||||
if (!user_mode(regs)) {
|
||||
rc = SIGSEGV;
|
||||
goto bail;
|
||||
|
|
|
@ -297,8 +297,8 @@ static inline int do_exception(struct pt_regs *regs, int access)
|
|||
* user context.
|
||||
*/
|
||||
fault = VM_FAULT_BADCONTEXT;
|
||||
if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm ||
|
||||
tsk->pagefault_disabled))
|
||||
if (unlikely(!user_space_fault(trans_exc_code) ||
|
||||
!mm || pagefault_disabled()))
|
||||
goto out;
|
||||
|
||||
address = trans_exc_code & __FAIL_ADDR_MASK;
|
||||
|
|
|
@ -73,7 +73,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long write,
|
|||
* If we're in an interrupt or have no user
|
||||
* context, we must not take the fault..
|
||||
*/
|
||||
if (in_atomic() || !mm || current->pagefault_disabled)
|
||||
if (!mm || pagefault_disabled())
|
||||
goto bad_area_nosemaphore;
|
||||
|
||||
if (user_mode(regs))
|
||||
|
|
|
@ -440,7 +440,7 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
|
|||
* If we're in an interrupt, have no user context or are running
|
||||
* in an atomic region then we must not take the fault:
|
||||
*/
|
||||
if (unlikely(in_atomic() || !mm || current->pagefault_disabled)) {
|
||||
if (unlikely(!mm || pagefault_disabled())) {
|
||||
bad_area_nosemaphore(regs, error_code, address);
|
||||
return;
|
||||
}
|
||||
|
|
|
@ -199,7 +199,7 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write,
|
|||
* If we're in an interrupt or have no user
|
||||
* context, we must not take the fault..
|
||||
*/
|
||||
if (in_atomic() || !mm || current->pagefault_disabled)
|
||||
if (!mm || pagefault_disabled())
|
||||
goto no_context;
|
||||
|
||||
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
|
||||
|
|
|
@ -332,7 +332,7 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs)
|
|||
* If we're in an interrupt or have no user
|
||||
* context, we must not take the fault..
|
||||
*/
|
||||
if (in_atomic() || !mm || current->pagefault_enabled)
|
||||
if (!mm || pagefault_disabled())
|
||||
goto intr_or_no_mm;
|
||||
|
||||
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
|
||||
|
|
|
@ -357,7 +357,7 @@ static int handle_page_fault(struct pt_regs *regs,
|
|||
* If we're in an interrupt, have no user context or are running in an
|
||||
* atomic region then we must not take the fault.
|
||||
*/
|
||||
if (in_atomic() || !mm || current->pagefault_disabled) {
|
||||
if (!mm || pagefault_disabled()) {
|
||||
vma = NULL; /* happy compiler */
|
||||
goto bad_area_nosemaphore;
|
||||
}
|
||||
|
|
|
@ -38,7 +38,7 @@ int handle_page_fault(unsigned long address, unsigned long ip,
|
|||
* If the fault was during atomic operation, don't take the fault, just
|
||||
* fail.
|
||||
*/
|
||||
if (in_atomic() || current->pagefault_disabled)
|
||||
if (pagefault_disabled())
|
||||
goto out_nosemaphore;
|
||||
|
||||
if (is_user)
|
||||
|
|
|
@ -1098,7 +1098,7 @@ __do_page_fault(struct pt_regs *regs, unsigned long error_code,
|
|||
* If we're in an interrupt, have no user context or are running
|
||||
* in an atomic region then we must not take the fault:
|
||||
*/
|
||||
if (unlikely(in_atomic() || !mm || current->pagefault_disabled)) {
|
||||
if (unlikely(!mm || pagefault_disabled())) {
|
||||
bad_area_nosemaphore(regs, error_code, address);
|
||||
return;
|
||||
}
|
||||
|
|
|
@ -57,7 +57,7 @@ void do_page_fault(struct pt_regs *regs)
|
|||
/* If we're in an interrupt or have no user
|
||||
* context, we must not take the fault..
|
||||
*/
|
||||
if (in_atomic() || !mm || current->pagefault_disabled) {
|
||||
if (!mm || pagefault_disabled()) {
|
||||
bad_page_fault(regs, address, SIGSEGV);
|
||||
return;
|
||||
}
|
||||
|
|
|
@ -54,6 +54,7 @@ struct sched_param {
|
|||
#include <linux/llist.h>
|
||||
#include <linux/uidgid.h>
|
||||
#include <linux/gfp.h>
|
||||
#include <linux/hardirq.h>
|
||||
|
||||
#include <asm/processor.h>
|
||||
|
||||
|
@ -1395,7 +1396,9 @@ struct task_struct {
|
|||
/* mutex deadlock detection */
|
||||
struct mutex_waiter *blocked_on;
|
||||
#endif
|
||||
#ifdef CONFIG_PREEMPT_RT_FULL
|
||||
int pagefault_disabled;
|
||||
#endif
|
||||
#ifdef CONFIG_TRACE_IRQFLAGS
|
||||
unsigned int irq_events;
|
||||
unsigned long hardirq_enable_ip;
|
||||
|
@ -1629,6 +1632,17 @@ static inline void task_numa_free(struct task_struct *p)
|
|||
}
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_PREEMPT_RT_FULL
|
||||
static inline bool cur_pf_disabled(void) { return current->pagefault_disabled; }
|
||||
#else
|
||||
static inline bool cur_pf_disabled(void) { return false; }
|
||||
#endif
|
||||
|
||||
static inline bool pagefault_disabled(void)
|
||||
{
|
||||
return in_atomic() || cur_pf_disabled();
|
||||
}
|
||||
|
||||
static inline struct pid *task_pid(struct task_struct *task)
|
||||
{
|
||||
return task->pids[PIDTYPE_PID].pid;
|
||||
|
|
|
@ -1303,7 +1303,9 @@ static struct task_struct *copy_process(unsigned long clone_flags,
|
|||
p->hardirq_context = 0;
|
||||
p->softirq_context = 0;
|
||||
#endif
|
||||
#ifdef CONFIG_PREEMPT_RT_FULL
|
||||
p->pagefault_disabled = 0;
|
||||
#endif
|
||||
#ifdef CONFIG_LOCKDEP
|
||||
p->lockdep_depth = 0; /* no locks held yet */
|
||||
p->curr_chain_key = 0;
|
||||
|
|
Loading…
Reference in New Issue