Merge branch 'arm64/vmap-stack' of git://git.kernel.org/pub/scm/linux/kernel/git/mark/linux into for-next/core
* 'arm64/vmap-stack' of git://git.kernel.org/pub/scm/linux/kernel/git/mark/linux: arm64: add VMAP_STACK overflow detection arm64: add on_accessible_stack() arm64: add basic VMAP_STACK support arm64: use an irq stack pointer arm64: assembler: allow adr_this_cpu to use the stack pointer arm64: factor out entry stack manipulation efi/arm64: add EFI_KIMG_ALIGN arm64: move SEGMENT_ALIGN to <asm/memory.h> arm64: clean up irq stack definitions arm64: clean up THREAD_* definitions arm64: factor out PAGE_* and CONT_* definitions arm64: kernel: remove {THREAD,IRQ_STACK}_START_SP fork: allow arch-override of VMAP stack alignment arm64: remove __die()'s stack dump
This commit is contained in:
commit
df5b95bee1
|
@ -75,6 +75,7 @@ config ARM64
|
|||
select HAVE_ARCH_SECCOMP_FILTER
|
||||
select HAVE_ARCH_TRACEHOOK
|
||||
select HAVE_ARCH_TRANSPARENT_HUGEPAGE
|
||||
select HAVE_ARCH_VMAP_STACK
|
||||
select HAVE_ARM_SMCCC
|
||||
select HAVE_EBPF_JIT
|
||||
select HAVE_C_RECORDMCOUNT
|
||||
|
|
|
@ -230,12 +230,18 @@ lr .req x30 // link register
|
|||
.endm
|
||||
|
||||
/*
|
||||
* @dst: Result of per_cpu(sym, smp_processor_id())
|
||||
* @dst: Result of per_cpu(sym, smp_processor_id()), can be SP for
|
||||
* non-module code
|
||||
* @sym: The name of the per-cpu variable
|
||||
* @tmp: scratch register
|
||||
*/
|
||||
.macro adr_this_cpu, dst, sym, tmp
|
||||
#ifndef MODULE
|
||||
adrp \tmp, \sym
|
||||
add \dst, \tmp, #:lo12:\sym
|
||||
#else
|
||||
adr_l \dst, \sym
|
||||
#endif
|
||||
mrs \tmp, tpidr_el1
|
||||
add \dst, \dst, \tmp
|
||||
.endm
|
||||
|
|
|
@ -4,6 +4,7 @@
|
|||
#include <asm/boot.h>
|
||||
#include <asm/cpufeature.h>
|
||||
#include <asm/io.h>
|
||||
#include <asm/memory.h>
|
||||
#include <asm/mmu_context.h>
|
||||
#include <asm/neon.h>
|
||||
#include <asm/ptrace.h>
|
||||
|
@ -48,6 +49,13 @@ int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md);
|
|||
*/
|
||||
#define EFI_FDT_ALIGN SZ_2M /* used by allocate_new_fdt_and_exit_boot() */
|
||||
|
||||
/*
|
||||
* In some configurations (e.g. VMAP_STACK && 64K pages), stacks built into the
|
||||
* kernel need greater alignment than we require the segments to be padded to.
|
||||
*/
|
||||
#define EFI_KIMG_ALIGN \
|
||||
(SEGMENT_ALIGN > THREAD_ALIGN ? SEGMENT_ALIGN : THREAD_ALIGN)
|
||||
|
||||
/* on arm64, the FDT may be located anywhere in system RAM */
|
||||
static inline unsigned long efi_get_max_fdt_addr(unsigned long dram_base)
|
||||
{
|
||||
|
|
|
@ -1,21 +1,12 @@
|
|||
#ifndef __ASM_IRQ_H
|
||||
#define __ASM_IRQ_H
|
||||
|
||||
#define IRQ_STACK_SIZE THREAD_SIZE
|
||||
#define IRQ_STACK_START_SP THREAD_START_SP
|
||||
|
||||
#ifndef __ASSEMBLER__
|
||||
|
||||
#include <linux/percpu.h>
|
||||
#include <linux/sched/task_stack.h>
|
||||
|
||||
#include <asm-generic/irq.h>
|
||||
#include <asm/thread_info.h>
|
||||
|
||||
struct pt_regs;
|
||||
|
||||
DECLARE_PER_CPU(unsigned long [IRQ_STACK_SIZE/sizeof(long)], irq_stack);
|
||||
|
||||
extern void set_handle_irq(void (*handle_irq)(struct pt_regs *));
|
||||
|
||||
static inline int nr_legacy_irqs(void)
|
||||
|
@ -23,21 +14,5 @@ static inline int nr_legacy_irqs(void)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static inline bool on_irq_stack(unsigned long sp)
|
||||
{
|
||||
unsigned long low = (unsigned long)raw_cpu_ptr(irq_stack);
|
||||
unsigned long high = low + IRQ_STACK_START_SP;
|
||||
|
||||
return (low <= sp && sp <= high);
|
||||
}
|
||||
|
||||
static inline bool on_task_stack(struct task_struct *tsk, unsigned long sp)
|
||||
{
|
||||
unsigned long low = (unsigned long)task_stack_page(tsk);
|
||||
unsigned long high = low + THREAD_SIZE;
|
||||
|
||||
return (low <= sp && sp < high);
|
||||
}
|
||||
|
||||
#endif /* !__ASSEMBLER__ */
|
||||
#endif
|
||||
|
|
|
@ -25,6 +25,7 @@
|
|||
#include <linux/const.h>
|
||||
#include <linux/types.h>
|
||||
#include <asm/bug.h>
|
||||
#include <asm/page-def.h>
|
||||
#include <asm/sizes.h>
|
||||
|
||||
/*
|
||||
|
@ -103,6 +104,58 @@
|
|||
#define KASAN_SHADOW_SIZE (0)
|
||||
#endif
|
||||
|
||||
#define MIN_THREAD_SHIFT 14
|
||||
|
||||
/*
|
||||
* VMAP'd stacks are allocated at page granularity, so we must ensure that such
|
||||
* stacks are a multiple of page size.
|
||||
*/
|
||||
#if defined(CONFIG_VMAP_STACK) && (MIN_THREAD_SHIFT < PAGE_SHIFT)
|
||||
#define THREAD_SHIFT PAGE_SHIFT
|
||||
#else
|
||||
#define THREAD_SHIFT MIN_THREAD_SHIFT
|
||||
#endif
|
||||
|
||||
#if THREAD_SHIFT >= PAGE_SHIFT
|
||||
#define THREAD_SIZE_ORDER (THREAD_SHIFT - PAGE_SHIFT)
|
||||
#endif
|
||||
|
||||
#define THREAD_SIZE (UL(1) << THREAD_SHIFT)
|
||||
|
||||
/*
|
||||
* By aligning VMAP'd stacks to 2 * THREAD_SIZE, we can detect overflow by
|
||||
* checking sp & (1 << THREAD_SHIFT), which we can do cheaply in the entry
|
||||
* assembly.
|
||||
*/
|
||||
#ifdef CONFIG_VMAP_STACK
|
||||
#define THREAD_ALIGN (2 * THREAD_SIZE)
|
||||
#else
|
||||
#define THREAD_ALIGN THREAD_SIZE
|
||||
#endif
|
||||
|
||||
#define IRQ_STACK_SIZE THREAD_SIZE
|
||||
|
||||
#define OVERFLOW_STACK_SIZE SZ_4K
|
||||
|
||||
/*
|
||||
* Alignment of kernel segments (e.g. .text, .data).
|
||||
*/
|
||||
#if defined(CONFIG_DEBUG_ALIGN_RODATA)
|
||||
/*
|
||||
* 4 KB granule: 1 level 2 entry
|
||||
* 16 KB granule: 128 level 3 entries, with contiguous bit
|
||||
* 64 KB granule: 32 level 3 entries, with contiguous bit
|
||||
*/
|
||||
#define SEGMENT_ALIGN SZ_2M
|
||||
#else
|
||||
/*
|
||||
* 4 KB granule: 16 level 3 entries, with contiguous bit
|
||||
* 16 KB granule: 4 level 3 entries, without contiguous bit
|
||||
* 64 KB granule: 1 level 3 entry
|
||||
*/
|
||||
#define SEGMENT_ALIGN SZ_64K
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Memory types available.
|
||||
*/
|
||||
|
|
|
@ -0,0 +1,34 @@
|
|||
/*
|
||||
* Based on arch/arm/include/asm/page.h
|
||||
*
|
||||
* Copyright (C) 1995-2003 Russell King
|
||||
* Copyright (C) 2017 ARM Ltd.
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License version 2 as
|
||||
* published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
#ifndef __ASM_PAGE_DEF_H
|
||||
#define __ASM_PAGE_DEF_H
|
||||
|
||||
#include <linux/const.h>
|
||||
|
||||
/* PAGE_SHIFT determines the page size */
|
||||
/* CONT_SHIFT determines the number of pages which can be tracked together */
|
||||
#define PAGE_SHIFT CONFIG_ARM64_PAGE_SHIFT
|
||||
#define CONT_SHIFT CONFIG_ARM64_CONT_SHIFT
|
||||
#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT)
|
||||
#define PAGE_MASK (~(PAGE_SIZE-1))
|
||||
|
||||
#define CONT_SIZE (_AC(1, UL) << (CONT_SHIFT + PAGE_SHIFT))
|
||||
#define CONT_MASK (~(CONT_SIZE-1))
|
||||
|
||||
#endif /* __ASM_PAGE_DEF_H */
|
|
@ -19,17 +19,7 @@
|
|||
#ifndef __ASM_PAGE_H
|
||||
#define __ASM_PAGE_H
|
||||
|
||||
#include <linux/const.h>
|
||||
|
||||
/* PAGE_SHIFT determines the page size */
|
||||
/* CONT_SHIFT determines the number of pages which can be tracked together */
|
||||
#define PAGE_SHIFT CONFIG_ARM64_PAGE_SHIFT
|
||||
#define CONT_SHIFT CONFIG_ARM64_CONT_SHIFT
|
||||
#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT)
|
||||
#define PAGE_MASK (~(PAGE_SIZE-1))
|
||||
|
||||
#define CONT_SIZE (_AC(1, UL) << (CONT_SHIFT + PAGE_SHIFT))
|
||||
#define CONT_MASK (~(CONT_SIZE-1))
|
||||
#include <asm/page-def.h>
|
||||
|
||||
#ifndef __ASSEMBLY__
|
||||
|
||||
|
|
|
@ -159,7 +159,7 @@ extern struct task_struct *cpu_switch_to(struct task_struct *prev,
|
|||
struct task_struct *next);
|
||||
|
||||
#define task_pt_regs(p) \
|
||||
((struct pt_regs *)(THREAD_START_SP + task_stack_page(p)) - 1)
|
||||
((struct pt_regs *)(THREAD_SIZE + task_stack_page(p)) - 1)
|
||||
|
||||
#define KSTK_EIP(tsk) ((unsigned long)task_pt_regs(tsk)->pc)
|
||||
#define KSTK_ESP(tsk) user_stack_pointer(task_pt_regs(tsk))
|
||||
|
|
|
@ -16,7 +16,12 @@
|
|||
#ifndef __ASM_STACKTRACE_H
|
||||
#define __ASM_STACKTRACE_H
|
||||
|
||||
struct task_struct;
|
||||
#include <linux/percpu.h>
|
||||
#include <linux/sched.h>
|
||||
#include <linux/sched/task_stack.h>
|
||||
|
||||
#include <asm/memory.h>
|
||||
#include <asm/ptrace.h>
|
||||
|
||||
struct stackframe {
|
||||
unsigned long fp;
|
||||
|
@ -31,4 +36,57 @@ extern void walk_stackframe(struct task_struct *tsk, struct stackframe *frame,
|
|||
int (*fn)(struct stackframe *, void *), void *data);
|
||||
extern void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk);
|
||||
|
||||
DECLARE_PER_CPU(unsigned long *, irq_stack_ptr);
|
||||
|
||||
static inline bool on_irq_stack(unsigned long sp)
|
||||
{
|
||||
unsigned long low = (unsigned long)raw_cpu_read(irq_stack_ptr);
|
||||
unsigned long high = low + IRQ_STACK_SIZE;
|
||||
|
||||
if (!low)
|
||||
return false;
|
||||
|
||||
return (low <= sp && sp < high);
|
||||
}
|
||||
|
||||
static inline bool on_task_stack(struct task_struct *tsk, unsigned long sp)
|
||||
{
|
||||
unsigned long low = (unsigned long)task_stack_page(tsk);
|
||||
unsigned long high = low + THREAD_SIZE;
|
||||
|
||||
return (low <= sp && sp < high);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_VMAP_STACK
|
||||
DECLARE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack);
|
||||
|
||||
static inline bool on_overflow_stack(unsigned long sp)
|
||||
{
|
||||
unsigned long low = (unsigned long)raw_cpu_ptr(overflow_stack);
|
||||
unsigned long high = low + OVERFLOW_STACK_SIZE;
|
||||
|
||||
return (low <= sp && sp < high);
|
||||
}
|
||||
#else
|
||||
static inline bool on_overflow_stack(unsigned long sp) { return false; }
|
||||
#endif
|
||||
|
||||
/*
|
||||
* We can only safely access per-cpu stacks from current in a non-preemptible
|
||||
* context.
|
||||
*/
|
||||
static inline bool on_accessible_stack(struct task_struct *tsk, unsigned long sp)
|
||||
{
|
||||
if (on_task_stack(tsk, sp))
|
||||
return true;
|
||||
if (tsk != current || preemptible())
|
||||
return false;
|
||||
if (on_irq_stack(sp))
|
||||
return true;
|
||||
if (on_overflow_stack(sp))
|
||||
return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
#endif /* __ASM_STACKTRACE_H */
|
||||
|
|
|
@ -23,19 +23,11 @@
|
|||
|
||||
#include <linux/compiler.h>
|
||||
|
||||
#ifdef CONFIG_ARM64_4K_PAGES
|
||||
#define THREAD_SIZE_ORDER 2
|
||||
#elif defined(CONFIG_ARM64_16K_PAGES)
|
||||
#define THREAD_SIZE_ORDER 0
|
||||
#endif
|
||||
|
||||
#define THREAD_SIZE 16384
|
||||
#define THREAD_START_SP (THREAD_SIZE - 16)
|
||||
|
||||
#ifndef __ASSEMBLY__
|
||||
|
||||
struct task_struct;
|
||||
|
||||
#include <asm/memory.h>
|
||||
#include <asm/stack_pointer.h>
|
||||
#include <asm/types.h>
|
||||
|
||||
|
|
|
@ -69,8 +69,55 @@
|
|||
#define BAD_FIQ 2
|
||||
#define BAD_ERROR 3
|
||||
|
||||
.macro kernel_entry, el, regsize = 64
|
||||
.macro kernel_ventry label
|
||||
.align 7
|
||||
sub sp, sp, #S_FRAME_SIZE
|
||||
#ifdef CONFIG_VMAP_STACK
|
||||
/*
|
||||
* Test whether the SP has overflowed, without corrupting a GPR.
|
||||
* Task and IRQ stacks are aligned to (1 << THREAD_SHIFT).
|
||||
*/
|
||||
add sp, sp, x0 // sp' = sp + x0
|
||||
sub x0, sp, x0 // x0' = sp' - x0 = (sp + x0) - x0 = sp
|
||||
tbnz x0, #THREAD_SHIFT, 0f
|
||||
sub x0, sp, x0 // x0'' = sp' - x0' = (sp + x0) - sp = x0
|
||||
sub sp, sp, x0 // sp'' = sp' - x0 = (sp + x0) - x0 = sp
|
||||
b \label
|
||||
|
||||
0:
|
||||
/*
|
||||
* Either we've just detected an overflow, or we've taken an exception
|
||||
* while on the overflow stack. Either way, we won't return to
|
||||
* userspace, and can clobber EL0 registers to free up GPRs.
|
||||
*/
|
||||
|
||||
/* Stash the original SP (minus S_FRAME_SIZE) in tpidr_el0. */
|
||||
msr tpidr_el0, x0
|
||||
|
||||
/* Recover the original x0 value and stash it in tpidrro_el0 */
|
||||
sub x0, sp, x0
|
||||
msr tpidrro_el0, x0
|
||||
|
||||
/* Switch to the overflow stack */
|
||||
adr_this_cpu sp, overflow_stack + OVERFLOW_STACK_SIZE, x0
|
||||
|
||||
/*
|
||||
* Check whether we were already on the overflow stack. This may happen
|
||||
* after panic() re-enables interrupts.
|
||||
*/
|
||||
mrs x0, tpidr_el0 // sp of interrupted context
|
||||
sub x0, sp, x0 // delta with top of overflow stack
|
||||
tst x0, #~(OVERFLOW_STACK_SIZE - 1) // within range?
|
||||
b.ne __bad_stack // no? -> bad stack pointer
|
||||
|
||||
/* We were already on the overflow stack. Restore sp/x0 and carry on. */
|
||||
sub sp, sp, x0
|
||||
mrs x0, tpidrro_el0
|
||||
#endif
|
||||
b \label
|
||||
.endm
|
||||
|
||||
.macro kernel_entry, el, regsize = 64
|
||||
.if \regsize == 32
|
||||
mov w0, w0 // zero upper 32 bits of x0
|
||||
.endif
|
||||
|
@ -269,8 +316,8 @@ alternative_else_nop_endif
|
|||
and x25, x25, #~(THREAD_SIZE - 1)
|
||||
cbnz x25, 9998f
|
||||
|
||||
adr_this_cpu x25, irq_stack, x26
|
||||
mov x26, #IRQ_STACK_START_SP
|
||||
ldr_this_cpu x25, irq_stack_ptr, x26
|
||||
mov x26, #IRQ_STACK_SIZE
|
||||
add x26, x25, x26
|
||||
|
||||
/* switch to the irq stack */
|
||||
|
@ -318,34 +365,62 @@ tsk .req x28 // current thread_info
|
|||
|
||||
.align 11
|
||||
ENTRY(vectors)
|
||||
ventry el1_sync_invalid // Synchronous EL1t
|
||||
ventry el1_irq_invalid // IRQ EL1t
|
||||
ventry el1_fiq_invalid // FIQ EL1t
|
||||
ventry el1_error_invalid // Error EL1t
|
||||
kernel_ventry el1_sync_invalid // Synchronous EL1t
|
||||
kernel_ventry el1_irq_invalid // IRQ EL1t
|
||||
kernel_ventry el1_fiq_invalid // FIQ EL1t
|
||||
kernel_ventry el1_error_invalid // Error EL1t
|
||||
|
||||
ventry el1_sync // Synchronous EL1h
|
||||
ventry el1_irq // IRQ EL1h
|
||||
ventry el1_fiq_invalid // FIQ EL1h
|
||||
ventry el1_error_invalid // Error EL1h
|
||||
kernel_ventry el1_sync // Synchronous EL1h
|
||||
kernel_ventry el1_irq // IRQ EL1h
|
||||
kernel_ventry el1_fiq_invalid // FIQ EL1h
|
||||
kernel_ventry el1_error_invalid // Error EL1h
|
||||
|
||||
ventry el0_sync // Synchronous 64-bit EL0
|
||||
ventry el0_irq // IRQ 64-bit EL0
|
||||
ventry el0_fiq_invalid // FIQ 64-bit EL0
|
||||
ventry el0_error_invalid // Error 64-bit EL0
|
||||
kernel_ventry el0_sync // Synchronous 64-bit EL0
|
||||
kernel_ventry el0_irq // IRQ 64-bit EL0
|
||||
kernel_ventry el0_fiq_invalid // FIQ 64-bit EL0
|
||||
kernel_ventry el0_error_invalid // Error 64-bit EL0
|
||||
|
||||
#ifdef CONFIG_COMPAT
|
||||
ventry el0_sync_compat // Synchronous 32-bit EL0
|
||||
ventry el0_irq_compat // IRQ 32-bit EL0
|
||||
ventry el0_fiq_invalid_compat // FIQ 32-bit EL0
|
||||
ventry el0_error_invalid_compat // Error 32-bit EL0
|
||||
kernel_ventry el0_sync_compat // Synchronous 32-bit EL0
|
||||
kernel_ventry el0_irq_compat // IRQ 32-bit EL0
|
||||
kernel_ventry el0_fiq_invalid_compat // FIQ 32-bit EL0
|
||||
kernel_ventry el0_error_invalid_compat // Error 32-bit EL0
|
||||
#else
|
||||
ventry el0_sync_invalid // Synchronous 32-bit EL0
|
||||
ventry el0_irq_invalid // IRQ 32-bit EL0
|
||||
ventry el0_fiq_invalid // FIQ 32-bit EL0
|
||||
ventry el0_error_invalid // Error 32-bit EL0
|
||||
kernel_ventry el0_sync_invalid // Synchronous 32-bit EL0
|
||||
kernel_ventry el0_irq_invalid // IRQ 32-bit EL0
|
||||
kernel_ventry el0_fiq_invalid // FIQ 32-bit EL0
|
||||
kernel_ventry el0_error_invalid // Error 32-bit EL0
|
||||
#endif
|
||||
END(vectors)
|
||||
|
||||
#ifdef CONFIG_VMAP_STACK
|
||||
/*
|
||||
* We detected an overflow in kernel_ventry, which switched to the
|
||||
* overflow stack. Stash the exception regs, and head to our overflow
|
||||
* handler.
|
||||
*/
|
||||
__bad_stack:
|
||||
/* Restore the original x0 value */
|
||||
mrs x0, tpidrro_el0
|
||||
|
||||
/*
|
||||
* Store the original GPRs to the new stack. The orginal SP (minus
|
||||
* S_FRAME_SIZE) was stashed in tpidr_el0 by kernel_ventry.
|
||||
*/
|
||||
sub sp, sp, #S_FRAME_SIZE
|
||||
kernel_entry 1
|
||||
mrs x0, tpidr_el0
|
||||
add x0, x0, #S_FRAME_SIZE
|
||||
str x0, [sp, #S_SP]
|
||||
|
||||
/* Stash the regs for handle_bad_stack */
|
||||
mov x0, sp
|
||||
|
||||
/* Time to die */
|
||||
bl handle_bad_stack
|
||||
ASM_BUG()
|
||||
#endif /* CONFIG_VMAP_STACK */
|
||||
|
||||
/*
|
||||
* Invalid mode handlers
|
||||
*/
|
||||
|
|
|
@ -23,15 +23,16 @@
|
|||
|
||||
#include <linux/kernel_stat.h>
|
||||
#include <linux/irq.h>
|
||||
#include <linux/memory.h>
|
||||
#include <linux/smp.h>
|
||||
#include <linux/init.h>
|
||||
#include <linux/irqchip.h>
|
||||
#include <linux/seq_file.h>
|
||||
#include <linux/vmalloc.h>
|
||||
|
||||
unsigned long irq_err_count;
|
||||
|
||||
/* irq stack only needs to be 16 byte aligned - not IRQ_STACK_SIZE aligned. */
|
||||
DEFINE_PER_CPU(unsigned long [IRQ_STACK_SIZE/sizeof(long)], irq_stack) __aligned(16);
|
||||
DEFINE_PER_CPU(unsigned long *, irq_stack_ptr);
|
||||
|
||||
int arch_show_interrupts(struct seq_file *p, int prec)
|
||||
{
|
||||
|
@ -50,8 +51,43 @@ void __init set_handle_irq(void (*handle_irq)(struct pt_regs *))
|
|||
handle_arch_irq = handle_irq;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_VMAP_STACK
|
||||
static void init_irq_stacks(void)
|
||||
{
|
||||
int cpu;
|
||||
unsigned long *p;
|
||||
|
||||
for_each_possible_cpu(cpu) {
|
||||
/*
|
||||
* To ensure that VMAP'd stack overflow detection works
|
||||
* correctly, the IRQ stacks need to have the same
|
||||
* alignment as other stacks.
|
||||
*/
|
||||
p = __vmalloc_node_range(IRQ_STACK_SIZE, THREAD_ALIGN,
|
||||
VMALLOC_START, VMALLOC_END,
|
||||
THREADINFO_GFP, PAGE_KERNEL,
|
||||
0, cpu_to_node(cpu),
|
||||
__builtin_return_address(0));
|
||||
|
||||
per_cpu(irq_stack_ptr, cpu) = p;
|
||||
}
|
||||
}
|
||||
#else
|
||||
/* irq stack only needs to be 16 byte aligned - not IRQ_STACK_SIZE aligned. */
|
||||
DEFINE_PER_CPU_ALIGNED(unsigned long [IRQ_STACK_SIZE/sizeof(long)], irq_stack);
|
||||
|
||||
static void init_irq_stacks(void)
|
||||
{
|
||||
int cpu;
|
||||
|
||||
for_each_possible_cpu(cpu)
|
||||
per_cpu(irq_stack_ptr, cpu) = per_cpu(irq_stack, cpu);
|
||||
}
|
||||
#endif
|
||||
|
||||
void __init init_IRQ(void)
|
||||
{
|
||||
init_irq_stacks();
|
||||
irqchip_init();
|
||||
if (!handle_arch_irq)
|
||||
panic("No interrupt controller found.");
|
||||
|
|
|
@ -42,6 +42,7 @@
|
|||
#include <asm/compat.h>
|
||||
#include <asm/debug-monitors.h>
|
||||
#include <asm/pgtable.h>
|
||||
#include <asm/stacktrace.h>
|
||||
#include <asm/syscall.h>
|
||||
#include <asm/traps.h>
|
||||
#include <asm/system_misc.h>
|
||||
|
|
|
@ -154,7 +154,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle)
|
|||
* page tables.
|
||||
*/
|
||||
secondary_data.task = idle;
|
||||
secondary_data.stack = task_stack_page(idle) + THREAD_START_SP;
|
||||
secondary_data.stack = task_stack_page(idle) + THREAD_SIZE;
|
||||
update_cpu_boot_status(CPU_MMU_OFF);
|
||||
__flush_dcache_area(&secondary_data, sizeof(secondary_data));
|
||||
|
||||
|
|
|
@ -50,12 +50,7 @@ int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame)
|
|||
if (!tsk)
|
||||
tsk = current;
|
||||
|
||||
/*
|
||||
* Switching between stacks is valid when tracing current and in
|
||||
* non-preemptible context.
|
||||
*/
|
||||
if (!(tsk == current && !preemptible() && on_irq_stack(fp)) &&
|
||||
!on_task_stack(tsk, fp))
|
||||
if (!on_accessible_stack(tsk, fp))
|
||||
return -EINVAL;
|
||||
|
||||
frame->fp = READ_ONCE_NOCHECK(*(unsigned long *)(fp));
|
||||
|
|
|
@ -32,6 +32,7 @@
|
|||
#include <linux/sched/signal.h>
|
||||
#include <linux/sched/debug.h>
|
||||
#include <linux/sched/task_stack.h>
|
||||
#include <linux/sizes.h>
|
||||
#include <linux/syscalls.h>
|
||||
#include <linux/mm_types.h>
|
||||
|
||||
|
@ -41,6 +42,7 @@
|
|||
#include <asm/esr.h>
|
||||
#include <asm/insn.h>
|
||||
#include <asm/traps.h>
|
||||
#include <asm/smp.h>
|
||||
#include <asm/stack_pointer.h>
|
||||
#include <asm/stacktrace.h>
|
||||
#include <asm/exception.h>
|
||||
|
@ -193,8 +195,7 @@ void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk)
|
|||
if (in_entry_text(frame.pc)) {
|
||||
stack = frame.fp - offsetof(struct pt_regs, stackframe);
|
||||
|
||||
if (on_task_stack(tsk, stack) ||
|
||||
(tsk == current && !preemptible() && on_irq_stack(stack)))
|
||||
if (on_accessible_stack(tsk, stack))
|
||||
dump_mem("", "Exception stack", stack,
|
||||
stack + sizeof(struct pt_regs));
|
||||
}
|
||||
|
@ -237,8 +238,6 @@ static int __die(const char *str, int err, struct pt_regs *regs)
|
|||
end_of_stack(tsk));
|
||||
|
||||
if (!user_mode(regs)) {
|
||||
dump_mem(KERN_EMERG, "Stack: ", regs->sp,
|
||||
THREAD_SIZE + (unsigned long)task_stack_page(tsk));
|
||||
dump_backtrace(regs, tsk);
|
||||
dump_instr(KERN_EMERG, regs);
|
||||
}
|
||||
|
@ -672,6 +671,43 @@ asmlinkage void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr)
|
|||
force_sig_info(info.si_signo, &info, current);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_VMAP_STACK
|
||||
|
||||
DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack)
|
||||
__aligned(16);
|
||||
|
||||
asmlinkage void handle_bad_stack(struct pt_regs *regs)
|
||||
{
|
||||
unsigned long tsk_stk = (unsigned long)current->stack;
|
||||
unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr);
|
||||
unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack);
|
||||
unsigned int esr = read_sysreg(esr_el1);
|
||||
unsigned long far = read_sysreg(far_el1);
|
||||
|
||||
console_verbose();
|
||||
pr_emerg("Insufficient stack space to handle exception!");
|
||||
|
||||
pr_emerg("ESR: 0x%08x -- %s\n", esr, esr_get_class_string(esr));
|
||||
pr_emerg("FAR: 0x%016lx\n", far);
|
||||
|
||||
pr_emerg("Task stack: [0x%016lx..0x%016lx]\n",
|
||||
tsk_stk, tsk_stk + THREAD_SIZE);
|
||||
pr_emerg("IRQ stack: [0x%016lx..0x%016lx]\n",
|
||||
irq_stk, irq_stk + THREAD_SIZE);
|
||||
pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n",
|
||||
ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE);
|
||||
|
||||
__show_regs(regs);
|
||||
|
||||
/*
|
||||
* We use nmi_panic to limit the potential for recusive overflows, and
|
||||
* to get a better stack trace.
|
||||
*/
|
||||
nmi_panic(NULL, "kernel stack overflow");
|
||||
cpu_park_loop();
|
||||
}
|
||||
#endif
|
||||
|
||||
void __pte_error(const char *file, int line, unsigned long val)
|
||||
{
|
||||
pr_err("%s:%d: bad pte %016lx.\n", file, line, val);
|
||||
|
|
|
@ -72,22 +72,6 @@ PECOFF_FILE_ALIGNMENT = 0x200;
|
|||
#define PECOFF_EDATA_PADDING
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_DEBUG_ALIGN_RODATA)
|
||||
/*
|
||||
* 4 KB granule: 1 level 2 entry
|
||||
* 16 KB granule: 128 level 3 entries, with contiguous bit
|
||||
* 64 KB granule: 32 level 3 entries, with contiguous bit
|
||||
*/
|
||||
#define SEGMENT_ALIGN SZ_2M
|
||||
#else
|
||||
/*
|
||||
* 4 KB granule: 16 level 3 entries, with contiguous bit
|
||||
* 16 KB granule: 4 level 3 entries, without contiguous bit
|
||||
* 64 KB granule: 1 level 3 entry
|
||||
*/
|
||||
#define SEGMENT_ALIGN SZ_64K
|
||||
#endif
|
||||
|
||||
SECTIONS
|
||||
{
|
||||
/*
|
||||
|
@ -192,7 +176,7 @@ SECTIONS
|
|||
|
||||
_data = .;
|
||||
_sdata = .;
|
||||
RW_DATA_SECTION(L1_CACHE_BYTES, PAGE_SIZE, THREAD_SIZE)
|
||||
RW_DATA_SECTION(L1_CACHE_BYTES, PAGE_SIZE, THREAD_ALIGN)
|
||||
|
||||
/*
|
||||
* Data written with the MMU off but read with the MMU on requires
|
||||
|
|
|
@ -11,6 +11,7 @@
|
|||
*/
|
||||
#include <linux/efi.h>
|
||||
#include <asm/efi.h>
|
||||
#include <asm/memory.h>
|
||||
#include <asm/sections.h>
|
||||
#include <asm/sysreg.h>
|
||||
|
||||
|
@ -81,9 +82,10 @@ efi_status_t handle_kernel_image(efi_system_table_t *sys_table_arg,
|
|||
/*
|
||||
* If CONFIG_DEBUG_ALIGN_RODATA is not set, produce a
|
||||
* displacement in the interval [0, MIN_KIMG_ALIGN) that
|
||||
* is a multiple of the minimal segment alignment (SZ_64K)
|
||||
* doesn't violate this kernel's de-facto alignment
|
||||
* constraints.
|
||||
*/
|
||||
u32 mask = (MIN_KIMG_ALIGN - 1) & ~(SZ_64K - 1);
|
||||
u32 mask = (MIN_KIMG_ALIGN - 1) & ~(EFI_KIMG_ALIGN - 1);
|
||||
u32 offset = !IS_ENABLED(CONFIG_DEBUG_ALIGN_RODATA) ?
|
||||
(phys_seed >> 32) & mask : TEXT_OFFSET;
|
||||
|
||||
|
|
|
@ -38,6 +38,10 @@ enum {
|
|||
|
||||
#ifdef __KERNEL__
|
||||
|
||||
#ifndef THREAD_ALIGN
|
||||
#define THREAD_ALIGN THREAD_SIZE
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_DEBUG_STACK_USAGE
|
||||
# define THREADINFO_GFP (GFP_KERNEL_ACCOUNT | __GFP_NOTRACK | \
|
||||
__GFP_ZERO)
|
||||
|
|
|
@ -88,6 +88,7 @@
|
|||
#include <linux/sysctl.h>
|
||||
#include <linux/kcov.h>
|
||||
#include <linux/livepatch.h>
|
||||
#include <linux/thread_info.h>
|
||||
|
||||
#include <asm/pgtable.h>
|
||||
#include <asm/pgalloc.h>
|
||||
|
@ -217,7 +218,7 @@ static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node)
|
|||
return s->addr;
|
||||
}
|
||||
|
||||
stack = __vmalloc_node_range(THREAD_SIZE, THREAD_SIZE,
|
||||
stack = __vmalloc_node_range(THREAD_SIZE, THREAD_ALIGN,
|
||||
VMALLOC_START, VMALLOC_END,
|
||||
THREADINFO_GFP,
|
||||
PAGE_KERNEL,
|
||||
|
|
Loading…
Reference in New Issue