aarch64: Implement -moutline-atomics

2020-04-01  Kyrylo Tkachov  <kyrylo.tkachov@arm.com>

	Backport from mainline
	2019-09-19  Richard Henderson  <richard.henderson@linaro.org>

	* config/aarch64/aarch64.opt (-moutline-atomics): New.
	* config/aarch64/aarch64.c (aarch64_atomic_ool_func): New.
	(aarch64_ool_cas_names, aarch64_ool_swp_names): New.
	(aarch64_ool_ldadd_names, aarch64_ool_ldset_names): New.
	(aarch64_ool_ldclr_names, aarch64_ool_ldeor_names): New.
	(aarch64_expand_compare_and_swap): Honor TARGET_OUTLINE_ATOMICS.
	* config/aarch64/atomics.md (atomic_exchange<ALLI>): Likewise.
	(atomic_<atomic_op><ALLI>): Likewise.
	(atomic_fetch_<atomic_op><ALLI>): Likewise.
	(atomic_<atomic_op>_fetch<ALLI>): Likewise.
	* doc/invoke.texi: Document -moutline-atomics.

	* gcc.target/aarch64/atomic-op-acq_rel.c: Use -mno-outline-atomics.
	* gcc.target/aarch64/atomic-comp-swap-release-acquire.c: Likewise.
	* gcc.target/aarch64/atomic-op-acquire.c: Likewise.
	* gcc.target/aarch64/atomic-op-char.c: Likewise.
	* gcc.target/aarch64/atomic-op-consume.c: Likewise.
	* gcc.target/aarch64/atomic-op-imm.c: Likewise.
	* gcc.target/aarch64/atomic-op-int.c: Likewise.
	* gcc.target/aarch64/atomic-op-long.c: Likewise.
	* gcc.target/aarch64/atomic-op-relaxed.c: Likewise.
	* gcc.target/aarch64/atomic-op-release.c: Likewise.
	* gcc.target/aarch64/atomic-op-seq_cst.c: Likewise.
	* gcc.target/aarch64/atomic-op-short.c: Likewise.
	* gcc.target/aarch64/atomic_cmp_exchange_zero_reg_1.c: Likewise.
	* gcc.target/aarch64/atomic_cmp_exchange_zero_strong_1.c: Likewise.
	* gcc.target/aarch64/sync-comp-swap.c: Likewise.
	* gcc.target/aarch64/sync-op-acquire.c: Likewise.
	* gcc.target/aarch64/sync-op-full.c: Likewise.
This commit is contained in:
Kyrylo Tkachov 2020-04-01 12:00:14 +01:00
parent b36f6a808e
commit 9cf2784a69
24 changed files with 261 additions and 26 deletions

View File

@ -1,3 +1,20 @@
2020-04-01 Kyrylo Tkachov <kyrylo.tkachov@arm.com>
Backport from mainline
2019-09-19 Richard Henderson <richard.henderson@linaro.org>
* config/aarch64/aarch64.opt (-moutline-atomics): New.
* config/aarch64/aarch64.c (aarch64_atomic_ool_func): New.
(aarch64_ool_cas_names, aarch64_ool_swp_names): New.
(aarch64_ool_ldadd_names, aarch64_ool_ldset_names): New.
(aarch64_ool_ldclr_names, aarch64_ool_ldeor_names): New.
(aarch64_expand_compare_and_swap): Honor TARGET_OUTLINE_ATOMICS.
* config/aarch64/atomics.md (atomic_exchange<ALLI>): Likewise.
(atomic_<atomic_op><ALLI>): Likewise.
(atomic_fetch_<atomic_op><ALLI>): Likewise.
(atomic_<atomic_op>_fetch<ALLI>): Likewise.
* doc/invoke.texi: Document -moutline-atomics.
2020-04-01 Kyrylo Tkachov <kyrylo.tkachov@arm.com>
Backport from mainline

View File

@ -644,4 +644,17 @@ poly_uint64 aarch64_regmode_natural_size (machine_mode);
bool aarch64_high_bits_all_ones_p (HOST_WIDE_INT);
struct atomic_ool_names
{
const char *str[5][4];
};
rtx aarch64_atomic_ool_func(machine_mode mode, rtx model_rtx,
const atomic_ool_names *names);
extern const atomic_ool_names aarch64_ool_swp_names;
extern const atomic_ool_names aarch64_ool_ldadd_names;
extern const atomic_ool_names aarch64_ool_ldset_names;
extern const atomic_ool_names aarch64_ool_ldclr_names;
extern const atomic_ool_names aarch64_ool_ldeor_names;
#endif /* GCC_AARCH64_PROTOS_H */

View File

@ -15482,6 +15482,82 @@ aarch64_emit_unlikely_jump (rtx insn)
add_reg_br_prob_note (jump, profile_probability::very_unlikely ());
}
/* We store the names of the various atomic helpers in a 5x4 array.
Return the libcall function given MODE, MODEL and NAMES. */
rtx
aarch64_atomic_ool_func(machine_mode mode, rtx model_rtx,
const atomic_ool_names *names)
{
memmodel model = memmodel_base (INTVAL (model_rtx));
int mode_idx, model_idx;
switch (mode)
{
case E_QImode:
mode_idx = 0;
break;
case E_HImode:
mode_idx = 1;
break;
case E_SImode:
mode_idx = 2;
break;
case E_DImode:
mode_idx = 3;
break;
case E_TImode:
mode_idx = 4;
break;
default:
gcc_unreachable ();
}
switch (model)
{
case MEMMODEL_RELAXED:
model_idx = 0;
break;
case MEMMODEL_CONSUME:
case MEMMODEL_ACQUIRE:
model_idx = 1;
break;
case MEMMODEL_RELEASE:
model_idx = 2;
break;
case MEMMODEL_ACQ_REL:
case MEMMODEL_SEQ_CST:
model_idx = 3;
break;
default:
gcc_unreachable ();
}
return init_one_libfunc_visibility (names->str[mode_idx][model_idx],
VISIBILITY_HIDDEN);
}
#define DEF0(B, N) \
{ "__aarch64_" #B #N "_relax", \
"__aarch64_" #B #N "_acq", \
"__aarch64_" #B #N "_rel", \
"__aarch64_" #B #N "_acq_rel" }
#define DEF4(B) DEF0(B, 1), DEF0(B, 2), DEF0(B, 4), DEF0(B, 8), \
{ NULL, NULL, NULL, NULL }
#define DEF5(B) DEF0(B, 1), DEF0(B, 2), DEF0(B, 4), DEF0(B, 8), DEF0(B, 16)
static const atomic_ool_names aarch64_ool_cas_names = { { DEF5(cas) } };
const atomic_ool_names aarch64_ool_swp_names = { { DEF4(swp) } };
const atomic_ool_names aarch64_ool_ldadd_names = { { DEF4(ldadd) } };
const atomic_ool_names aarch64_ool_ldset_names = { { DEF4(ldset) } };
const atomic_ool_names aarch64_ool_ldclr_names = { { DEF4(ldclr) } };
const atomic_ool_names aarch64_ool_ldeor_names = { { DEF4(ldeor) } };
#undef DEF0
#undef DEF4
#undef DEF5
/* Expand a compare and swap pattern. */
void
@ -15528,6 +15604,17 @@ aarch64_expand_compare_and_swap (rtx operands[])
newval, mod_s));
cc_reg = aarch64_gen_compare_reg_maybe_ze (NE, rval, oldval, mode);
}
else if (TARGET_OUTLINE_ATOMICS)
{
/* Oldval must satisfy compare afterward. */
if (!aarch64_plus_operand (oldval, mode))
oldval = force_reg (mode, oldval);
rtx func = aarch64_atomic_ool_func (mode, mod_s, &aarch64_ool_cas_names);
rval = emit_library_call_value (func, NULL_RTX, LCT_NORMAL, r_mode,
oldval, mode, newval, mode,
XEXP (mem, 0), Pmode);
cc_reg = aarch64_gen_compare_reg_maybe_ze (NE, rval, oldval, mode);
}
else
{
/* The oldval predicate varies by mode. Test it and force to reg. */

View File

@ -255,3 +255,6 @@ user-land code.
TargetVariable
long aarch64_stack_protector_guard_offset = 0
moutline-atomics
Target Report Mask(OUTLINE_ATOMICS) Save
Generate local calls to out-of-line atomic operations.

View File

@ -186,16 +186,27 @@
(match_operand:SI 3 "const_int_operand" "")]
""
{
rtx (*gen) (rtx, rtx, rtx, rtx);
/* Use an atomic SWP when available. */
if (TARGET_LSE)
gen = gen_aarch64_atomic_exchange<mode>_lse;
{
emit_insn (gen_aarch64_atomic_exchange<mode>_lse
(operands[0], operands[1], operands[2], operands[3]));
}
else if (TARGET_OUTLINE_ATOMICS)
{
machine_mode mode = <MODE>mode;
rtx func = aarch64_atomic_ool_func (mode, operands[3],
&aarch64_ool_swp_names);
rtx rval = emit_library_call_value (func, operands[0], LCT_NORMAL,
mode, operands[2], mode,
XEXP (operands[1], 0), Pmode);
emit_move_insn (operands[0], rval);
}
else
gen = gen_aarch64_atomic_exchange<mode>;
emit_insn (gen (operands[0], operands[1], operands[2], operands[3]));
{
emit_insn (gen_aarch64_atomic_exchange<mode>
(operands[0], operands[1], operands[2], operands[3]));
}
DONE;
}
)
@ -280,6 +291,39 @@
}
operands[1] = force_reg (<MODE>mode, operands[1]);
}
else if (TARGET_OUTLINE_ATOMICS)
{
const atomic_ool_names *names;
switch (<CODE>)
{
case MINUS:
operands[1] = expand_simple_unop (<MODE>mode, NEG, operands[1],
NULL, 1);
/* fallthru */
case PLUS:
names = &aarch64_ool_ldadd_names;
break;
case IOR:
names = &aarch64_ool_ldset_names;
break;
case XOR:
names = &aarch64_ool_ldeor_names;
break;
case AND:
operands[1] = expand_simple_unop (<MODE>mode, NOT, operands[1],
NULL, 1);
names = &aarch64_ool_ldclr_names;
break;
default:
gcc_unreachable ();
}
machine_mode mode = <MODE>mode;
rtx func = aarch64_atomic_ool_func (mode, operands[2], names);
emit_library_call_value (func, NULL_RTX, LCT_NORMAL, mode,
operands[1], mode,
XEXP (operands[0], 0), Pmode);
DONE;
}
else
gen = gen_aarch64_atomic_<atomic_optab><mode>;
@ -405,6 +449,40 @@
}
operands[2] = force_reg (<MODE>mode, operands[2]);
}
else if (TARGET_OUTLINE_ATOMICS)
{
const atomic_ool_names *names;
switch (<CODE>)
{
case MINUS:
operands[2] = expand_simple_unop (<MODE>mode, NEG, operands[2],
NULL, 1);
/* fallthru */
case PLUS:
names = &aarch64_ool_ldadd_names;
break;
case IOR:
names = &aarch64_ool_ldset_names;
break;
case XOR:
names = &aarch64_ool_ldeor_names;
break;
case AND:
operands[2] = expand_simple_unop (<MODE>mode, NOT, operands[2],
NULL, 1);
names = &aarch64_ool_ldclr_names;
break;
default:
gcc_unreachable ();
}
machine_mode mode = <MODE>mode;
rtx func = aarch64_atomic_ool_func (mode, operands[3], names);
rtx rval = emit_library_call_value (func, operands[0], LCT_NORMAL, mode,
operands[2], mode,
XEXP (operands[1], 0), Pmode);
emit_move_insn (operands[0], rval);
DONE;
}
else
gen = gen_aarch64_atomic_fetch_<atomic_optab><mode>;
@ -494,7 +572,7 @@
{
/* Use an atomic load-operate instruction when possible. In this case
we will re-compute the result from the original mem value. */
if (TARGET_LSE)
if (TARGET_LSE || TARGET_OUTLINE_ATOMICS)
{
rtx tmp = gen_reg_rtx (<MODE>mode);
operands[2] = force_reg (<MODE>mode, operands[2]);

View File

@ -637,7 +637,8 @@ Objective-C and Objective-C++ Dialects}.
-march=@var{name} -mcpu=@var{name} -mtune=@var{name} @gol
-moverride=@var{string} -mverbose-cost-dump @gol
-mstack-protector-guard=@var{guard} -mstack-protector-guard-reg=@var{sysreg} @gol
-mstack-protector-guard-offset=@var{offset} -mtrack-speculation }
-mstack-protector-guard-offset=@var{offset} -mtrack-speculation @gol
-moutline-atomics }
@emph{Adapteva Epiphany Options}
@gccoptlist{-mhalf-reg-file -mprefer-short-insn-regs @gol
@ -15782,6 +15783,19 @@ be used by the compiler when expanding calls to
@code{__builtin_speculation_safe_copy} to permit a more efficient code
sequence to be generated.
@item -moutline-atomics
@itemx -mno-outline-atomics
Enable or disable calls to out-of-line helpers to implement atomic operations.
These helpers will, at runtime, determine if the LSE instructions from
ARMv8.1-A can be used; if not, they will use the load/store-exclusive
instructions that are present in the base ARMv8.0 ISA.
This option is only applicable when compiling for the base ARMv8.0
instruction set. If using a later revision, e.g. @option{-march=armv8.1-a}
or @option{-march=armv8-a+lse}, the ARMv8.1-Atomics instructions will be
used directly. The same applies when using @option{-mcpu=} when the
selected cpu supports the @samp{lse} feature.
@item -march=@var{name}
@opindex march
Specify the name of the target architecture and, optionally, one or

View File

@ -1,3 +1,26 @@
2020-04-01 Kyrylo Tkachov <kyrylo.tkachov@arm.com>
Backport from mainline
2019-09-19 Richard Henderson <richard.henderson@linaro.org>
* gcc.target/aarch64/atomic-op-acq_rel.c: Use -mno-outline-atomics.
* gcc.target/aarch64/atomic-comp-swap-release-acquire.c: Likewise.
* gcc.target/aarch64/atomic-op-acquire.c: Likewise.
* gcc.target/aarch64/atomic-op-char.c: Likewise.
* gcc.target/aarch64/atomic-op-consume.c: Likewise.
* gcc.target/aarch64/atomic-op-imm.c: Likewise.
* gcc.target/aarch64/atomic-op-int.c: Likewise.
* gcc.target/aarch64/atomic-op-long.c: Likewise.
* gcc.target/aarch64/atomic-op-relaxed.c: Likewise.
* gcc.target/aarch64/atomic-op-release.c: Likewise.
* gcc.target/aarch64/atomic-op-seq_cst.c: Likewise.
* gcc.target/aarch64/atomic-op-short.c: Likewise.
* gcc.target/aarch64/atomic_cmp_exchange_zero_reg_1.c: Likewise.
* gcc.target/aarch64/atomic_cmp_exchange_zero_strong_1.c: Likewise.
* gcc.target/aarch64/sync-comp-swap.c: Likewise.
* gcc.target/aarch64/sync-op-acquire.c: Likewise.
* gcc.target/aarch64/sync-op-full.c: Likewise.
2020-03-28 Tobias Burnus <tobias@codesourcery.com>
Backport from mainline

View File

@ -1,5 +1,5 @@
/* { dg-do compile } */
/* { dg-options "-march=armv8-a+nolse -O2 -fno-ipa-icf" } */
/* { dg-options "-march=armv8-a+nolse -O2 -fno-ipa-icf -mno-outline-atomics" } */
#include "atomic-comp-swap-release-acquire.x"

View File

@ -1,5 +1,5 @@
/* { dg-do compile } */
/* { dg-options "-march=armv8-a+nolse -O2" } */
/* { dg-options "-march=armv8-a+nolse -O2 -mno-outline-atomics" } */
#include "atomic-op-acq_rel.x"

View File

@ -1,5 +1,5 @@
/* { dg-do compile } */
/* { dg-options "-march=armv8-a+nolse -O2" } */
/* { dg-options "-march=armv8-a+nolse -O2 -mno-outline-atomics" } */
#include "atomic-op-acquire.x"

View File

@ -1,5 +1,5 @@
/* { dg-do compile } */
/* { dg-options "-march=armv8-a+nolse -O2" } */
/* { dg-options "-march=armv8-a+nolse -O2 -mno-outline-atomics" } */
#include "atomic-op-char.x"

View File

@ -1,5 +1,5 @@
/* { dg-do compile } */
/* { dg-options "-march=armv8-a+nolse -O2" } */
/* { dg-options "-march=armv8-a+nolse -O2 -mno-outline-atomics" } */
#include "atomic-op-consume.x"

View File

@ -1,5 +1,5 @@
/* { dg-do compile } */
/* { dg-options "-march=armv8-a+nolse -O2" } */
/* { dg-options "-march=armv8-a+nolse -O2 -mno-outline-atomics" } */
int v = 0;

View File

@ -1,5 +1,5 @@
/* { dg-do compile } */
/* { dg-options "-march=armv8-a+nolse -O2" } */
/* { dg-options "-march=armv8-a+nolse -O2 -mno-outline-atomics" } */
#include "atomic-op-int.x"

View File

@ -1,5 +1,5 @@
/* { dg-do compile } */
/* { dg-options "-march=armv8-a+nolse -O2" } */
/* { dg-options "-march=armv8-a+nolse -O2 -mno-outline-atomics" } */
long v = 0;

View File

@ -1,5 +1,5 @@
/* { dg-do compile } */
/* { dg-options "-march=armv8-a+nolse -O2" } */
/* { dg-options "-march=armv8-a+nolse -O2 -mno-outline-atomics" } */
#include "atomic-op-relaxed.x"

View File

@ -1,5 +1,5 @@
/* { dg-do compile } */
/* { dg-options "-march=armv8-a+nolse -O2" } */
/* { dg-options "-march=armv8-a+nolse -O2 -mno-outline-atomics" } */
#include "atomic-op-release.x"

View File

@ -1,5 +1,5 @@
/* { dg-do compile } */
/* { dg-options "-march=armv8-a+nolse -O2" } */
/* { dg-options "-march=armv8-a+nolse -O2 -mno-outline-atomics" } */
#include "atomic-op-seq_cst.x"

View File

@ -1,5 +1,5 @@
/* { dg-do compile } */
/* { dg-options "-march=armv8-a+nolse -O2" } */
/* { dg-options "-march=armv8-a+nolse -O2 -mno-outline-atomics" } */
#include "atomic-op-short.x"

View File

@ -1,5 +1,5 @@
/* { dg-do compile } */
/* { dg-options "-O2 -march=armv8-a+nolse" } */
/* { dg-options "-O2 -march=armv8-a+nolse -mno-outline-atomics" } */
/* { dg-skip-if "" { *-*-* } { "-mcpu=*" } { "" } } */
int

View File

@ -1,5 +1,5 @@
/* { dg-do compile } */
/* { dg-options "-O2 -march=armv8-a+nolse" } */
/* { dg-options "-O2 -march=armv8-a+nolse -mno-outline-atomics" } */
/* { dg-skip-if "" { *-*-* } { "-mcpu=*" } { "" } } */
int

View File

@ -1,5 +1,5 @@
/* { dg-do compile } */
/* { dg-options "-march=armv8-a+nolse -O2 -fno-ipa-icf" } */
/* { dg-options "-march=armv8-a+nolse -O2 -fno-ipa-icf -mno-outline-atomics" } */
#include "sync-comp-swap.x"

View File

@ -1,5 +1,5 @@
/* { dg-do compile } */
/* { dg-options "-march=armv8-a+nolse -O2" } */
/* { dg-options "-march=armv8-a+nolse -O2 -mno-outline-atomics" } */
#include "sync-op-acquire.x"

View File

@ -1,5 +1,5 @@
/* { dg-do compile } */
/* { dg-options "-march=armv8-a+nolse -O2" } */
/* { dg-options "-march=armv8-a+nolse -O2 -mno-outline-atomics" } */
#include "sync-op-full.x"