aarch64: Add support for unpacked SVE MLA and MAD

This patch extends the MLA/MAD patterns to support unpacked
integer vectors.  The type suffix could be either the element
size or the container size, but using the element size should
be more efficient.

gcc/
	* config/aarch64/aarch64-sve.md (fma<mode>4): Extend from SVE_FULL_I
	to SVE_I.
	(@aarch64_pred_fma<mode>, cond_fma<mode>, *cond_fma<mode>_2)
	(*cond_fma<mode>_4, *cond_fma<mode>_any): Likewise.

gcc/testsuite/
	* gcc.target/aarch64/sve/mla_2.c: New test.
	* g++.target/aarch64/sve/cond_mla_1.C: Likewise.
	* g++.target/aarch64/sve/cond_mla_2.C: Likewise.
	* g++.target/aarch64/sve/cond_mla_3.C: Likewise.
	* g++.target/aarch64/sve/cond_mla_4.C: Likewise.
	* g++.target/aarch64/sve/cond_mla_5.C: Likewise.
This commit is contained in:
Richard Sandiford 2021-01-13 13:00:12 +00:00
parent 3ddc18251a
commit cf7a335306
7 changed files with 246 additions and 44 deletions

View File

@ -6554,15 +6554,15 @@
;; Unpredicated integer addition of product.
(define_expand "fma<mode>4"
[(set (match_operand:SVE_FULL_I 0 "register_operand")
(plus:SVE_FULL_I
(unspec:SVE_FULL_I
[(set (match_operand:SVE_I 0 "register_operand")
(plus:SVE_I
(unspec:SVE_I
[(match_dup 4)
(mult:SVE_FULL_I
(match_operand:SVE_FULL_I 1 "register_operand")
(match_operand:SVE_FULL_I 2 "nonmemory_operand"))]
(mult:SVE_I
(match_operand:SVE_I 1 "register_operand")
(match_operand:SVE_I 2 "nonmemory_operand"))]
UNSPEC_PRED_X)
(match_operand:SVE_FULL_I 3 "register_operand")))]
(match_operand:SVE_I 3 "register_operand")))]
"TARGET_SVE"
{
if (aarch64_prepare_sve_int_fma (operands, PLUS))
@ -6573,15 +6573,15 @@
;; Predicated integer addition of product.
(define_insn "@aarch64_pred_fma<mode>"
[(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, w, ?&w")
(plus:SVE_FULL_I
(unspec:SVE_FULL_I
[(set (match_operand:SVE_I 0 "register_operand" "=w, w, ?&w")
(plus:SVE_I
(unspec:SVE_I
[(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl")
(mult:SVE_FULL_I
(match_operand:SVE_FULL_I 2 "register_operand" "%0, w, w")
(match_operand:SVE_FULL_I 3 "register_operand" "w, w, w"))]
(mult:SVE_I
(match_operand:SVE_I 2 "register_operand" "%0, w, w")
(match_operand:SVE_I 3 "register_operand" "w, w, w"))]
UNSPEC_PRED_X)
(match_operand:SVE_FULL_I 4 "register_operand" "w, 0, w")))]
(match_operand:SVE_I 4 "register_operand" "w, 0, w")))]
"TARGET_SVE"
"@
mad\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>
@ -6592,15 +6592,15 @@
;; Predicated integer addition of product with merging.
(define_expand "cond_fma<mode>"
[(set (match_operand:SVE_FULL_I 0 "register_operand")
(unspec:SVE_FULL_I
[(set (match_operand:SVE_I 0 "register_operand")
(unspec:SVE_I
[(match_operand:<VPRED> 1 "register_operand")
(plus:SVE_FULL_I
(mult:SVE_FULL_I
(match_operand:SVE_FULL_I 2 "register_operand")
(match_operand:SVE_FULL_I 3 "general_operand"))
(match_operand:SVE_FULL_I 4 "register_operand"))
(match_operand:SVE_FULL_I 5 "aarch64_simd_reg_or_zero")]
(plus:SVE_I
(mult:SVE_I
(match_operand:SVE_I 2 "register_operand")
(match_operand:SVE_I 3 "general_operand"))
(match_operand:SVE_I 4 "register_operand"))
(match_operand:SVE_I 5 "aarch64_simd_reg_or_zero")]
UNSPEC_SEL))]
"TARGET_SVE"
{
@ -6615,14 +6615,14 @@
;; Predicated integer addition of product, merging with the first input.
(define_insn "*cond_fma<mode>_2"
[(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
(unspec:SVE_FULL_I
[(set (match_operand:SVE_I 0 "register_operand" "=w, ?&w")
(unspec:SVE_I
[(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
(plus:SVE_FULL_I
(mult:SVE_FULL_I
(match_operand:SVE_FULL_I 2 "register_operand" "0, w")
(match_operand:SVE_FULL_I 3 "register_operand" "w, w"))
(match_operand:SVE_FULL_I 4 "register_operand" "w, w"))
(plus:SVE_I
(mult:SVE_I
(match_operand:SVE_I 2 "register_operand" "0, w")
(match_operand:SVE_I 3 "register_operand" "w, w"))
(match_operand:SVE_I 4 "register_operand" "w, w"))
(match_dup 2)]
UNSPEC_SEL))]
"TARGET_SVE"
@ -6634,14 +6634,14 @@
;; Predicated integer addition of product, merging with the third input.
(define_insn "*cond_fma<mode>_4"
[(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w")
(unspec:SVE_FULL_I
[(set (match_operand:SVE_I 0 "register_operand" "=w, ?&w")
(unspec:SVE_I
[(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
(plus:SVE_FULL_I
(mult:SVE_FULL_I
(match_operand:SVE_FULL_I 2 "register_operand" "w, w")
(match_operand:SVE_FULL_I 3 "register_operand" "w, w"))
(match_operand:SVE_FULL_I 4 "register_operand" "0, w"))
(plus:SVE_I
(mult:SVE_I
(match_operand:SVE_I 2 "register_operand" "w, w")
(match_operand:SVE_I 3 "register_operand" "w, w"))
(match_operand:SVE_I 4 "register_operand" "0, w"))
(match_dup 4)]
UNSPEC_SEL))]
"TARGET_SVE"
@ -6653,15 +6653,15 @@
;; Predicated integer addition of product, merging with an independent value.
(define_insn_and_rewrite "*cond_fma<mode>_any"
[(set (match_operand:SVE_FULL_I 0 "register_operand" "=&w, &w, &w, &w, &w, ?&w")
(unspec:SVE_FULL_I
[(set (match_operand:SVE_I 0 "register_operand" "=&w, &w, &w, &w, &w, ?&w")
(unspec:SVE_I
[(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl, Upl, Upl")
(plus:SVE_FULL_I
(mult:SVE_FULL_I
(match_operand:SVE_FULL_I 2 "register_operand" "w, w, 0, w, w, w")
(match_operand:SVE_FULL_I 3 "register_operand" "w, w, w, 0, w, w"))
(match_operand:SVE_FULL_I 4 "register_operand" "w, 0, w, w, w, w"))
(match_operand:SVE_FULL_I 5 "aarch64_simd_reg_or_zero" "Dz, Dz, Dz, Dz, 0, w")]
(plus:SVE_I
(mult:SVE_I
(match_operand:SVE_I 2 "register_operand" "w, w, 0, w, w, w")
(match_operand:SVE_I 3 "register_operand" "w, w, w, 0, w, w"))
(match_operand:SVE_I 4 "register_operand" "w, 0, w, w, w, w"))
(match_operand:SVE_I 5 "aarch64_simd_reg_or_zero" "Dz, Dz, Dz, Dz, 0, w")]
UNSPEC_SEL))]
"TARGET_SVE
&& !rtx_equal_p (operands[2], operands[5])

View File

@ -0,0 +1,33 @@
/* { dg-do assemble { target aarch64_asm_sve_ok } } */
/* { dg-options "-O -msve-vector-bits=2048 -save-temps" } */
#include <stdint.h>
#define TEST_OP(TYPE) \
TYPE \
test##_##TYPE##_reg (TYPE a, TYPE b, TYPE c, TYPE d) \
{ \
return d == 0 ? a + b * c : a; \
}
#define TEST_TYPE(TYPE, SIZE) \
typedef TYPE TYPE##SIZE __attribute__((vector_size(SIZE))); \
TEST_OP (TYPE##SIZE)
TEST_TYPE (uint8_t, 32)
TEST_TYPE (uint8_t, 64)
TEST_TYPE (uint16_t, 64)
TEST_TYPE (uint8_t, 128)
TEST_TYPE (uint16_t, 128)
TEST_TYPE (uint32_t, 128)
/* { dg-final { scan-assembler {\tld1b\t(z[0-9]+)\.h, p[0-7]/z, \[x0\][^L]*\tmla\t\1\.b,} } } */
/* { dg-final { scan-assembler {\tld1b\t(z[0-9]+)\.s, p[0-7]/z, \[x0\][^L]*\tmla\t\1\.b,} } } */
/* { dg-final { scan-assembler {\tld1b\t(z[0-9]+)\.d, p[0-7]/z, \[x0\][^L]*\tmla\t\1\.b,} } } */
/* { dg-final { scan-assembler {\tld1h\t(z[0-9]+)\.s, p[0-7]/z, \[x0\][^L]*\tmla\t\1\.h,} } } */
/* { dg-final { scan-assembler {\tld1h\t(z[0-9]+)\.d, p[0-7]/z, \[x0\][^L]*\tmla\t\1\.h,} } } */
/* { dg-final { scan-assembler {\tld1w\t(z[0-9]+)\.d, p[0-7]/z, \[x0\][^L]*\tmla\t\1\.s,} } } */
/* { dg-final { scan-assembler-not {\tsel\t} } } */

View File

@ -0,0 +1,33 @@
/* { dg-do assemble { target aarch64_asm_sve_ok } } */
/* { dg-options "-O -msve-vector-bits=2048 -save-temps" } */
#include <stdint.h>
#define TEST_OP(TYPE) \
TYPE \
test##_##TYPE##_reg (TYPE a, TYPE b, TYPE c, TYPE d) \
{ \
return d == 0 ? a + b * c : b; \
}
#define TEST_TYPE(TYPE, SIZE) \
typedef TYPE TYPE##SIZE __attribute__((vector_size(SIZE))); \
TEST_OP (TYPE##SIZE)
TEST_TYPE (uint8_t, 32)
TEST_TYPE (uint8_t, 64)
TEST_TYPE (uint16_t, 64)
TEST_TYPE (uint8_t, 128)
TEST_TYPE (uint16_t, 128)
TEST_TYPE (uint32_t, 128)
/* { dg-final { scan-assembler {\tld1b\t(z[0-9]+)\.h, p[0-7]/z, \[x1\][^L]*\tmad\t\1\.b,} } } */
/* { dg-final { scan-assembler {\tld1b\t(z[0-9]+)\.s, p[0-7]/z, \[x1\][^L]*\tmad\t\1\.b,} } } */
/* { dg-final { scan-assembler {\tld1b\t(z[0-9]+)\.d, p[0-7]/z, \[x1\][^L]*\tmad\t\1\.b,} } } */
/* { dg-final { scan-assembler {\tld1h\t(z[0-9]+)\.s, p[0-7]/z, \[x1\][^L]*\tmad\t\1\.h,} } } */
/* { dg-final { scan-assembler {\tld1h\t(z[0-9]+)\.d, p[0-7]/z, \[x1\][^L]*\tmad\t\1\.h,} } } */
/* { dg-final { scan-assembler {\tld1w\t(z[0-9]+)\.d, p[0-7]/z, \[x1\][^L]*\tmad\t\1\.s,} } } */
/* { dg-final { scan-assembler-not {\tsel\t} } } */

View File

@ -0,0 +1,33 @@
/* { dg-do assemble { target aarch64_asm_sve_ok } } */
/* { dg-options "-O2 -msve-vector-bits=2048 -save-temps" } */
#include <stdint.h>
#define TEST_OP(TYPE) \
TYPE \
test##_##TYPE##_reg (TYPE a, TYPE b, TYPE c, TYPE d) \
{ \
return d == 0 ? a + b * c : c; \
}
#define TEST_TYPE(TYPE, SIZE) \
typedef TYPE TYPE##SIZE __attribute__((vector_size(SIZE))); \
TEST_OP (TYPE##SIZE)
TEST_TYPE (uint8_t, 32)
TEST_TYPE (uint8_t, 64)
TEST_TYPE (uint16_t, 64)
TEST_TYPE (uint8_t, 128)
TEST_TYPE (uint16_t, 128)
TEST_TYPE (uint32_t, 128)
/* { dg-final { scan-assembler {\tld1b\t(z[0-9]+)\.h, p[0-7]/z, \[x2\][^L]*\tmad\t\1\.b,} } } */
/* { dg-final { scan-assembler {\tld1b\t(z[0-9]+)\.s, p[0-7]/z, \[x2\][^L]*\tmad\t\1\.b,} } } */
/* { dg-final { scan-assembler {\tld1b\t(z[0-9]+)\.d, p[0-7]/z, \[x2\][^L]*\tmad\t\1\.b,} } } */
/* { dg-final { scan-assembler {\tld1h\t(z[0-9]+)\.s, p[0-7]/z, \[x2\][^L]*\tmad\t\1\.h,} } } */
/* { dg-final { scan-assembler {\tld1h\t(z[0-9]+)\.d, p[0-7]/z, \[x2\][^L]*\tmad\t\1\.h,} } } */
/* { dg-final { scan-assembler {\tld1w\t(z[0-9]+)\.d, p[0-7]/z, \[x2\][^L]*\tmad\t\1\.s,} } } */
/* { dg-final { scan-assembler-not {\tsel\t} } } */

View File

@ -0,0 +1,36 @@
/* { dg-do assemble { target aarch64_asm_sve_ok } } */
/* { dg-options "-O -msve-vector-bits=2048 -save-temps" } */
#include <stdint.h>
#define TEST_OP(TYPE) \
TYPE \
test##_##TYPE##_reg (TYPE a, TYPE b, TYPE c, TYPE d) \
{ \
return d == 0 ? a + b * c : d; \
}
#define TEST_TYPE(TYPE, SIZE) \
typedef TYPE TYPE##SIZE __attribute__((vector_size(SIZE))); \
TEST_OP (TYPE##SIZE)
TEST_TYPE (uint8_t, 32)
TEST_TYPE (uint8_t, 64)
TEST_TYPE (uint16_t, 64)
TEST_TYPE (uint8_t, 128)
TEST_TYPE (uint16_t, 128)
TEST_TYPE (uint32_t, 128)
/* { dg-final { scan-assembler {\tld1b\t(z[0-9]+)\.h, p[0-7]/z, \[x3\][^L]*\tmla\t\1\.b,} } } */
/* { dg-final { scan-assembler {\tld1b\t(z[0-9]+)\.s, p[0-7]/z, \[x3\][^L]*\tmla\t\1\.b,} } } */
/* { dg-final { scan-assembler {\tld1b\t(z[0-9]+)\.d, p[0-7]/z, \[x3\][^L]*\tmla\t\1\.b,} } } */
/* { dg-final { scan-assembler {\tld1h\t(z[0-9]+)\.s, p[0-7]/z, \[x3\][^L]*\tmla\t\1\.h,} } } */
/* { dg-final { scan-assembler {\tld1h\t(z[0-9]+)\.d, p[0-7]/z, \[x3\][^L]*\tmla\t\1\.h,} } } */
/* { dg-final { scan-assembler {\tld1w\t(z[0-9]+)\.d, p[0-7]/z, \[x3\][^L]*\tmla\t\1\.s,} } } */
/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.b, p[0-7]/m, z[0-9]+\.b\n} 3 } } */
/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
/* { dg-final { scan-assembler-not {\tsel\t} } } */

View File

@ -0,0 +1,33 @@
/* { dg-do assemble { target aarch64_asm_sve_ok } } */
/* { dg-options "-O -msve-vector-bits=2048 -save-temps" } */
#include <stdint.h>
#define TEST_OP(TYPE) \
TYPE \
test##_##TYPE##_reg (TYPE a, TYPE b, TYPE c, TYPE d) \
{ \
return d == 0 ? a + b * c : 0; \
}
#define TEST_TYPE(TYPE, SIZE) \
typedef TYPE TYPE##SIZE __attribute__((vector_size(SIZE))); \
TEST_OP (TYPE##SIZE)
TEST_TYPE (uint8_t, 32)
TEST_TYPE (uint8_t, 64)
TEST_TYPE (uint16_t, 64)
TEST_TYPE (uint8_t, 128)
TEST_TYPE (uint16_t, 128)
TEST_TYPE (uint32_t, 128)
/* { dg-final { scan-assembler-times {\t(?:mla|mad)\tz[0-9]+\.b,} 3 } } */
/* { dg-final { scan-assembler-times {\t(?:mla|mad)\tz[0-9]+\.h,} 2 } } */
/* { dg-final { scan-assembler-times {\t(?:mla|mad)\tz[0-9]+\.s,} 1 } } */
/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.b, p[0-7]/z, z[0-9]+\.b\n} 3 } } */
/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 2 } } */
/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/z, z[0-9]+\.s\n} 1 } } */
/* { dg-final { scan-assembler-not {\tsel\t} } } */

View File

@ -0,0 +1,34 @@
/* { dg-do assemble { target aarch64_asm_sve_ok } } */
/* { dg-options "-O -msve-vector-bits=2048 -save-temps" } */
#include <stdint.h>
#define TEST_OP(TYPE) \
TYPE \
test##_##TYPE##_##AMT (TYPE a, TYPE b, TYPE c) \
{ \
return a + b * c; \
}
#define TEST_TYPE(TYPE, SIZE) \
typedef TYPE TYPE##SIZE __attribute__((vector_size(SIZE))); \
TEST_OP (TYPE##SIZE)
TEST_TYPE (int8_t, 32)
TEST_TYPE (uint8_t, 32)
TEST_TYPE (int8_t, 64)
TEST_TYPE (uint8_t, 64)
TEST_TYPE (int16_t, 64)
TEST_TYPE (uint16_t, 64)
TEST_TYPE (int8_t, 128)
TEST_TYPE (uint8_t, 128)
TEST_TYPE (int16_t, 128)
TEST_TYPE (uint16_t, 128)
TEST_TYPE (int32_t, 128)
TEST_TYPE (uint32_t, 128)
/* { dg-final { scan-assembler-times {\t(?:mla|mad)\tz[0-9]+\.b,} 6 } } */
/* { dg-final { scan-assembler-times {\t(?:mla|mad)\tz[0-9]+\.h,} 4 } } */
/* { dg-final { scan-assembler-times {\t(?:mla|mad)\tz[0-9]+\.s,} 2 } } */