86289a4ff4
Merged revision: 82bc6a094e85014f1891ef9407496f44af8fe442 with the fix for PR sanitizer/102911
447 lines
11 KiB
ArmAsm
447 lines
11 KiB
ArmAsm
// The content of this file is x86_64-only:
|
|
#if defined(__x86_64__)
|
|
|
|
#include "sanitizer_common/sanitizer_asm.h"
|
|
|
|
#if !defined(__APPLE__)
|
|
.section .text
|
|
#else
|
|
.section __TEXT,__text
|
|
#endif
|
|
|
|
ASM_HIDDEN(__tsan_trace_switch)
|
|
.globl ASM_SYMBOL(__tsan_trace_switch_thunk)
|
|
ASM_SYMBOL(__tsan_trace_switch_thunk):
|
|
CFI_STARTPROC
|
|
_CET_ENDBR
|
|
# Save scratch registers.
|
|
push %rax
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
CFI_REL_OFFSET(%rax, 0)
|
|
push %rcx
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
CFI_REL_OFFSET(%rcx, 0)
|
|
push %rdx
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
CFI_REL_OFFSET(%rdx, 0)
|
|
push %rsi
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
CFI_REL_OFFSET(%rsi, 0)
|
|
push %rdi
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
CFI_REL_OFFSET(%rdi, 0)
|
|
push %r8
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
CFI_REL_OFFSET(%r8, 0)
|
|
push %r9
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
CFI_REL_OFFSET(%r9, 0)
|
|
push %r10
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
CFI_REL_OFFSET(%r10, 0)
|
|
push %r11
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
CFI_REL_OFFSET(%r11, 0)
|
|
# All XMM registers are caller-saved.
|
|
sub $0x100, %rsp
|
|
CFI_ADJUST_CFA_OFFSET(0x100)
|
|
vmovdqu %xmm0, 0x0(%rsp)
|
|
vmovdqu %xmm1, 0x10(%rsp)
|
|
vmovdqu %xmm2, 0x20(%rsp)
|
|
vmovdqu %xmm3, 0x30(%rsp)
|
|
vmovdqu %xmm4, 0x40(%rsp)
|
|
vmovdqu %xmm5, 0x50(%rsp)
|
|
vmovdqu %xmm6, 0x60(%rsp)
|
|
vmovdqu %xmm7, 0x70(%rsp)
|
|
vmovdqu %xmm8, 0x80(%rsp)
|
|
vmovdqu %xmm9, 0x90(%rsp)
|
|
vmovdqu %xmm10, 0xa0(%rsp)
|
|
vmovdqu %xmm11, 0xb0(%rsp)
|
|
vmovdqu %xmm12, 0xc0(%rsp)
|
|
vmovdqu %xmm13, 0xd0(%rsp)
|
|
vmovdqu %xmm14, 0xe0(%rsp)
|
|
vmovdqu %xmm15, 0xf0(%rsp)
|
|
# Align stack frame.
|
|
push %rbx # non-scratch
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
CFI_REL_OFFSET(%rbx, 0)
|
|
mov %rsp, %rbx # save current rsp
|
|
CFI_DEF_CFA_REGISTER(%rbx)
|
|
shr $4, %rsp # clear 4 lsb, align to 16
|
|
shl $4, %rsp
|
|
|
|
call ASM_SYMBOL(__tsan_trace_switch)
|
|
|
|
# Unalign stack frame back.
|
|
mov %rbx, %rsp # restore the original rsp
|
|
CFI_DEF_CFA_REGISTER(%rsp)
|
|
pop %rbx
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
# Restore scratch registers.
|
|
vmovdqu 0x0(%rsp), %xmm0
|
|
vmovdqu 0x10(%rsp), %xmm1
|
|
vmovdqu 0x20(%rsp), %xmm2
|
|
vmovdqu 0x30(%rsp), %xmm3
|
|
vmovdqu 0x40(%rsp), %xmm4
|
|
vmovdqu 0x50(%rsp), %xmm5
|
|
vmovdqu 0x60(%rsp), %xmm6
|
|
vmovdqu 0x70(%rsp), %xmm7
|
|
vmovdqu 0x80(%rsp), %xmm8
|
|
vmovdqu 0x90(%rsp), %xmm9
|
|
vmovdqu 0xa0(%rsp), %xmm10
|
|
vmovdqu 0xb0(%rsp), %xmm11
|
|
vmovdqu 0xc0(%rsp), %xmm12
|
|
vmovdqu 0xd0(%rsp), %xmm13
|
|
vmovdqu 0xe0(%rsp), %xmm14
|
|
vmovdqu 0xf0(%rsp), %xmm15
|
|
add $0x100, %rsp
|
|
CFI_ADJUST_CFA_OFFSET(-0x100)
|
|
pop %r11
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
pop %r10
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
pop %r9
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
pop %r8
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
pop %rdi
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
pop %rsi
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
pop %rdx
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
pop %rcx
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
pop %rax
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
CFI_RESTORE(%rax)
|
|
CFI_RESTORE(%rbx)
|
|
CFI_RESTORE(%rcx)
|
|
CFI_RESTORE(%rdx)
|
|
CFI_RESTORE(%rsi)
|
|
CFI_RESTORE(%rdi)
|
|
CFI_RESTORE(%r8)
|
|
CFI_RESTORE(%r9)
|
|
CFI_RESTORE(%r10)
|
|
CFI_RESTORE(%r11)
|
|
ret
|
|
CFI_ENDPROC
|
|
|
|
ASM_HIDDEN(__tsan_report_race)
|
|
.globl ASM_SYMBOL(__tsan_report_race_thunk)
|
|
ASM_SYMBOL(__tsan_report_race_thunk):
|
|
CFI_STARTPROC
|
|
_CET_ENDBR
|
|
# Save scratch registers.
|
|
push %rax
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
CFI_REL_OFFSET(%rax, 0)
|
|
push %rcx
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
CFI_REL_OFFSET(%rcx, 0)
|
|
push %rdx
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
CFI_REL_OFFSET(%rdx, 0)
|
|
push %rsi
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
CFI_REL_OFFSET(%rsi, 0)
|
|
push %rdi
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
CFI_REL_OFFSET(%rdi, 0)
|
|
push %r8
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
CFI_REL_OFFSET(%r8, 0)
|
|
push %r9
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
CFI_REL_OFFSET(%r9, 0)
|
|
push %r10
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
CFI_REL_OFFSET(%r10, 0)
|
|
push %r11
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
CFI_REL_OFFSET(%r11, 0)
|
|
# All XMM registers are caller-saved.
|
|
sub $0x100, %rsp
|
|
CFI_ADJUST_CFA_OFFSET(0x100)
|
|
vmovdqu %xmm0, 0x0(%rsp)
|
|
vmovdqu %xmm1, 0x10(%rsp)
|
|
vmovdqu %xmm2, 0x20(%rsp)
|
|
vmovdqu %xmm3, 0x30(%rsp)
|
|
vmovdqu %xmm4, 0x40(%rsp)
|
|
vmovdqu %xmm5, 0x50(%rsp)
|
|
vmovdqu %xmm6, 0x60(%rsp)
|
|
vmovdqu %xmm7, 0x70(%rsp)
|
|
vmovdqu %xmm8, 0x80(%rsp)
|
|
vmovdqu %xmm9, 0x90(%rsp)
|
|
vmovdqu %xmm10, 0xa0(%rsp)
|
|
vmovdqu %xmm11, 0xb0(%rsp)
|
|
vmovdqu %xmm12, 0xc0(%rsp)
|
|
vmovdqu %xmm13, 0xd0(%rsp)
|
|
vmovdqu %xmm14, 0xe0(%rsp)
|
|
vmovdqu %xmm15, 0xf0(%rsp)
|
|
# Align stack frame.
|
|
push %rbx # non-scratch
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
CFI_REL_OFFSET(%rbx, 0)
|
|
mov %rsp, %rbx # save current rsp
|
|
CFI_DEF_CFA_REGISTER(%rbx)
|
|
shr $4, %rsp # clear 4 lsb, align to 16
|
|
shl $4, %rsp
|
|
|
|
call ASM_SYMBOL(__tsan_report_race)
|
|
|
|
# Unalign stack frame back.
|
|
mov %rbx, %rsp # restore the original rsp
|
|
CFI_DEF_CFA_REGISTER(%rsp)
|
|
pop %rbx
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
# Restore scratch registers.
|
|
vmovdqu 0x0(%rsp), %xmm0
|
|
vmovdqu 0x10(%rsp), %xmm1
|
|
vmovdqu 0x20(%rsp), %xmm2
|
|
vmovdqu 0x30(%rsp), %xmm3
|
|
vmovdqu 0x40(%rsp), %xmm4
|
|
vmovdqu 0x50(%rsp), %xmm5
|
|
vmovdqu 0x60(%rsp), %xmm6
|
|
vmovdqu 0x70(%rsp), %xmm7
|
|
vmovdqu 0x80(%rsp), %xmm8
|
|
vmovdqu 0x90(%rsp), %xmm9
|
|
vmovdqu 0xa0(%rsp), %xmm10
|
|
vmovdqu 0xb0(%rsp), %xmm11
|
|
vmovdqu 0xc0(%rsp), %xmm12
|
|
vmovdqu 0xd0(%rsp), %xmm13
|
|
vmovdqu 0xe0(%rsp), %xmm14
|
|
vmovdqu 0xf0(%rsp), %xmm15
|
|
add $0x100, %rsp
|
|
CFI_ADJUST_CFA_OFFSET(-0x100)
|
|
pop %r11
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
pop %r10
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
pop %r9
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
pop %r8
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
pop %rdi
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
pop %rsi
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
pop %rdx
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
pop %rcx
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
pop %rax
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
CFI_RESTORE(%rax)
|
|
CFI_RESTORE(%rbx)
|
|
CFI_RESTORE(%rcx)
|
|
CFI_RESTORE(%rdx)
|
|
CFI_RESTORE(%rsi)
|
|
CFI_RESTORE(%rdi)
|
|
CFI_RESTORE(%r8)
|
|
CFI_RESTORE(%r9)
|
|
CFI_RESTORE(%r10)
|
|
CFI_RESTORE(%r11)
|
|
ret
|
|
CFI_ENDPROC
|
|
|
|
ASM_HIDDEN(__tsan_setjmp)
|
|
#if defined(__NetBSD__)
|
|
.comm _ZN14__interception15real___setjmp14E,8,8
|
|
#elif !defined(__APPLE__)
|
|
.comm _ZN14__interception11real_setjmpE,8,8
|
|
#endif
|
|
#if defined(__NetBSD__)
|
|
.globl ASM_SYMBOL_INTERCEPTOR(__setjmp14)
|
|
ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__setjmp14))
|
|
ASM_SYMBOL_INTERCEPTOR(__setjmp14):
|
|
#else
|
|
.globl ASM_SYMBOL_INTERCEPTOR(setjmp)
|
|
ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp))
|
|
ASM_SYMBOL_INTERCEPTOR(setjmp):
|
|
#endif
|
|
CFI_STARTPROC
|
|
_CET_ENDBR
|
|
// save env parameter
|
|
push %rdi
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
CFI_REL_OFFSET(%rdi, 0)
|
|
// obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
|
|
#if defined(__FreeBSD__) || defined(__NetBSD__)
|
|
lea 8(%rsp), %rdi
|
|
#elif defined(__linux__) || defined(__APPLE__)
|
|
lea 16(%rsp), %rdi
|
|
#else
|
|
# error "Unknown platform"
|
|
#endif
|
|
// call tsan interceptor
|
|
call ASM_SYMBOL(__tsan_setjmp)
|
|
// restore env parameter
|
|
pop %rdi
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
CFI_RESTORE(%rdi)
|
|
// tail jump to libc setjmp
|
|
movl $0, %eax
|
|
#if defined(__NetBSD__)
|
|
movq _ZN14__interception15real___setjmp14E@GOTPCREL(%rip), %rdx
|
|
jmp *(%rdx)
|
|
#elif !defined(__APPLE__)
|
|
movq _ZN14__interception11real_setjmpE@GOTPCREL(%rip), %rdx
|
|
jmp *(%rdx)
|
|
#else
|
|
jmp ASM_SYMBOL(setjmp)
|
|
#endif
|
|
CFI_ENDPROC
|
|
#if defined(__NetBSD__)
|
|
ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__setjmp14))
|
|
#else
|
|
ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp))
|
|
#endif
|
|
|
|
.comm _ZN14__interception12real__setjmpE,8,8
|
|
.globl ASM_SYMBOL_INTERCEPTOR(_setjmp)
|
|
ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp))
|
|
ASM_SYMBOL_INTERCEPTOR(_setjmp):
|
|
CFI_STARTPROC
|
|
_CET_ENDBR
|
|
// save env parameter
|
|
push %rdi
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
CFI_REL_OFFSET(%rdi, 0)
|
|
// obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
|
|
#if defined(__FreeBSD__) || defined(__NetBSD__)
|
|
lea 8(%rsp), %rdi
|
|
#elif defined(__linux__) || defined(__APPLE__)
|
|
lea 16(%rsp), %rdi
|
|
#else
|
|
# error "Unknown platform"
|
|
#endif
|
|
// call tsan interceptor
|
|
call ASM_SYMBOL(__tsan_setjmp)
|
|
// restore env parameter
|
|
pop %rdi
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
CFI_RESTORE(%rdi)
|
|
// tail jump to libc setjmp
|
|
movl $0, %eax
|
|
#if !defined(__APPLE__)
|
|
movq _ZN14__interception12real__setjmpE@GOTPCREL(%rip), %rdx
|
|
jmp *(%rdx)
|
|
#else
|
|
jmp ASM_SYMBOL(_setjmp)
|
|
#endif
|
|
CFI_ENDPROC
|
|
ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp))
|
|
|
|
#if defined(__NetBSD__)
|
|
.comm _ZN14__interception18real___sigsetjmp14E,8,8
|
|
.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14)
|
|
ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14))
|
|
ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14):
|
|
#else
|
|
.comm _ZN14__interception14real_sigsetjmpE,8,8
|
|
.globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp)
|
|
ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
|
|
ASM_SYMBOL_INTERCEPTOR(sigsetjmp):
|
|
#endif
|
|
CFI_STARTPROC
|
|
_CET_ENDBR
|
|
// save env parameter
|
|
push %rdi
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
CFI_REL_OFFSET(%rdi, 0)
|
|
// save savesigs parameter
|
|
push %rsi
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
CFI_REL_OFFSET(%rsi, 0)
|
|
// align stack frame
|
|
sub $8, %rsp
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
// obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
|
|
#if defined(__FreeBSD__) || defined(__NetBSD__)
|
|
lea 24(%rsp), %rdi
|
|
#elif defined(__linux__) || defined(__APPLE__)
|
|
lea 32(%rsp), %rdi
|
|
#else
|
|
# error "Unknown platform"
|
|
#endif
|
|
// call tsan interceptor
|
|
call ASM_SYMBOL(__tsan_setjmp)
|
|
// unalign stack frame
|
|
add $8, %rsp
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
// restore savesigs parameter
|
|
pop %rsi
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
CFI_RESTORE(%rsi)
|
|
// restore env parameter
|
|
pop %rdi
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
CFI_RESTORE(%rdi)
|
|
// tail jump to libc sigsetjmp
|
|
movl $0, %eax
|
|
#if defined(__NetBSD__)
|
|
movq _ZN14__interception18real___sigsetjmp14E@GOTPCREL(%rip), %rdx
|
|
jmp *(%rdx)
|
|
#elif !defined(__APPLE__)
|
|
movq _ZN14__interception14real_sigsetjmpE@GOTPCREL(%rip), %rdx
|
|
jmp *(%rdx)
|
|
#else
|
|
jmp ASM_SYMBOL(sigsetjmp)
|
|
#endif
|
|
CFI_ENDPROC
|
|
#if defined(__NetBSD__)
|
|
ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp14))
|
|
#else
|
|
ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp))
|
|
#endif
|
|
|
|
#if !defined(__APPLE__) && !defined(__NetBSD__)
|
|
.comm _ZN14__interception16real___sigsetjmpE,8,8
|
|
.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)
|
|
ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
|
|
ASM_SYMBOL_INTERCEPTOR(__sigsetjmp):
|
|
CFI_STARTPROC
|
|
_CET_ENDBR
|
|
// save env parameter
|
|
push %rdi
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
CFI_REL_OFFSET(%rdi, 0)
|
|
// save savesigs parameter
|
|
push %rsi
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
CFI_REL_OFFSET(%rsi, 0)
|
|
// align stack frame
|
|
sub $8, %rsp
|
|
CFI_ADJUST_CFA_OFFSET(8)
|
|
// obtain SP, store in %rdi, first argument to `void __tsan_setjmp(uptr sp)`
|
|
#if defined(__FreeBSD__)
|
|
lea 24(%rsp), %rdi
|
|
#else
|
|
lea 32(%rsp), %rdi
|
|
#endif
|
|
// call tsan interceptor
|
|
call ASM_SYMBOL(__tsan_setjmp)
|
|
// unalign stack frame
|
|
add $8, %rsp
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
// restore savesigs parameter
|
|
pop %rsi
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
CFI_RESTORE(%rsi)
|
|
// restore env parameter
|
|
pop %rdi
|
|
CFI_ADJUST_CFA_OFFSET(-8)
|
|
CFI_RESTORE(%rdi)
|
|
// tail jump to libc sigsetjmp
|
|
movl $0, %eax
|
|
movq _ZN14__interception16real___sigsetjmpE@GOTPCREL(%rip), %rdx
|
|
jmp *(%rdx)
|
|
CFI_ENDPROC
|
|
ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
|
|
#endif // !defined(__APPLE__) && !defined(__NetBSD__)
|
|
|
|
NO_EXEC_STACK_DIRECTIVE
|
|
|
|
#endif
|