OpenE2K/qemu-e2k
13
4
Fork 0
Code Issues 4 Pull Requests Projects 2 Releases Activity

target/arm: Implement SME MOVA

Browse Source 
We can reuse the SVE functions for implementing moves to/from
horizontal tile slices, but we need new ones for moves to/from
vertical tile slices.

Reviewed-by: Peter Maydell <peter.maydell@linaro.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
Message-id: 20220708151540.18136-20-richard.henderson@linaro.org
Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
This commit is contained in:
Richard Henderson 2022-07-08 20:45:14 +05:30 committed by Peter Maydell
parent ad939afbfa
commit e9ad3ef19e
8 changed files with 331 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
target/arm/helper-sme.h
View File

@ -21,3 +21,15 @@ DEF_HELPER_FLAGS_2(set_pstate_sm, TCG_CALL_NO_RWG, void, env, i32)
DEF_HELPER_FLAGS_2(set_pstate_za, TCG_CALL_NO_RWG, void, env, i32)
DEF_HELPER_FLAGS_3(sme_zero, TCG_CALL_NO_RWG, void, env, i32, i32)
/* Move to/from vertical array slices, i.e. columns, so 'c'. */
DEF_HELPER_FLAGS_4(sme_mova_cz_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sme_mova_zc_b, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sme_mova_cz_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sme_mova_zc_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sme_mova_cz_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sme_mova_zc_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sme_mova_cz_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sme_mova_zc_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sme_mova_cz_q, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sme_mova_zc_q, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)

2
target/arm/helper-sve.h
View File

@ -325,6 +325,8 @@ DEF_HELPER_FLAGS_5(sve_sel_zpzz_s, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve_sel_zpzz_d, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve_sel_zpzz_q, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_5(sve2_addp_zpzz_b, TCG_CALL_NO_RWG,
void, ptr, ptr, ptr, ptr, i32)

15
target/arm/sme.decode
View File

@ -22,3 +22,18 @@
### SME Misc
ZERO 11000000 00 001 00000000000 imm:8
### SME Move into/from Array
%mova_rs 13:2 !function=plus_12
&mova esz rs pg zr za_imm v:bool to_vec:bool
MOVA 11000000 esz:2 00000 0 v:1 .. pg:3 zr:5 0 za_imm:4 \
&mova to_vec=0 rs=%mova_rs
MOVA 11000000 11 00000 1 v:1 .. pg:3 zr:5 0 za_imm:4 \
&mova to_vec=0 rs=%mova_rs esz=4
MOVA 11000000 esz:2 00001 0 v:1 .. pg:3 0 za_imm:4 zr:5 \
&mova to_vec=1 rs=%mova_rs
MOVA 11000000 11 00001 1 v:1 .. pg:3 0 za_imm:4 zr:5 \
&mova to_vec=1 rs=%mova_rs esz=4

151
target/arm/sme_helper.c
View File

@ -19,8 +19,10 @@
#include "qemu/osdep.h"
#include "cpu.h"
#include "internals.h"
#include "tcg/tcg-gvec-desc.h"
#include "exec/helper-proto.h"
#include "qemu/int128.h"
#include "vec_internal.h"
/* ResetSVEState */
void arm_reset_sve_state(CPUARMState *env)
@ -84,3 +86,150 @@ void helper_sme_zero(CPUARMState *env, uint32_t imm, uint32_t svl)
}
}
}
/*
* When considering the ZA storage as an array of elements of
* type T, the index within that array of the Nth element of
* a vertical slice of a tile can be calculated like this,
* regardless of the size of type T. This is because the tiles
* are interleaved, so if type T is size N bytes then row 1 of
* the tile is N rows away from row 0. The division by N to
* convert a byte offset into an array index and the multiplication
* by N to convert from vslice-index-within-the-tile to
* the index within the ZA storage cancel out.
*/
#define tile_vslice_index(i) ((i) * sizeof(ARMVectorReg))
/*
* When doing byte arithmetic on the ZA storage, the element
* byteoff bytes away in a tile vertical slice is always this
* many bytes away in the ZA storage, regardless of the
* size of the tile element, assuming that byteoff is a multiple
* of the element size. Again this is because of the interleaving
* of the tiles. For instance if we have 1 byte per element then
* each row of the ZA storage has one byte of the vslice data,
* and (counting from 0) byte 8 goes in row 8 of the storage
* at offset (8 * row-size-in-bytes).
* If we have 8 bytes per element then each row of the ZA storage
* has 8 bytes of the data, but there are 8 interleaved tiles and
* so byte 8 of the data goes into row 1 of the tile,
* which is again row 8 of the storage, so the offset is still
* (8 * row-size-in-bytes). Similarly for other element sizes.
*/
#define tile_vslice_offset(byteoff) ((byteoff) * sizeof(ARMVectorReg))
/*
* Move Zreg vector to ZArray column.
*/
#define DO_MOVA_C(NAME, TYPE, H) \
void HELPER(NAME)(void *za, void *vn, void *vg, uint32_t desc) \
{ \
int i, oprsz = simd_oprsz(desc); \
for (i = 0; i < oprsz; ) { \
uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
do { \
if (pg & 1) { \
*(TYPE *)(za + tile_vslice_offset(i)) = *(TYPE *)(vn + H(i)); \
} \
i += sizeof(TYPE); \
pg >>= sizeof(TYPE); \
} while (i & 15); \
} \
}
DO_MOVA_C(sme_mova_cz_b, uint8_t, H1)
DO_MOVA_C(sme_mova_cz_h, uint16_t, H1_2)
DO_MOVA_C(sme_mova_cz_s, uint32_t, H1_4)
void HELPER(sme_mova_cz_d)(void *za, void *vn, void *vg, uint32_t desc)
{
int i, oprsz = simd_oprsz(desc) / 8;
uint8_t *pg = vg;
uint64_t *n = vn;
uint64_t *a = za;
for (i = 0; i < oprsz; i++) {
if (pg[H1(i)] & 1) {
a[tile_vslice_index(i)] = n[i];
}
}
}
void HELPER(sme_mova_cz_q)(void *za, void *vn, void *vg, uint32_t desc)
{
int i, oprsz = simd_oprsz(desc) / 16;
uint16_t *pg = vg;
Int128 *n = vn;
Int128 *a = za;
/*
* Int128 is used here simply to copy 16 bytes, and to simplify
* the address arithmetic.
*/
for (i = 0; i < oprsz; i++) {
if (pg[H2(i)] & 1) {
a[tile_vslice_index(i)] = n[i];
}
}
}
#undef DO_MOVA_C
/*
* Move ZArray column to Zreg vector.
*/
#define DO_MOVA_Z(NAME, TYPE, H) \
void HELPER(NAME)(void *vd, void *za, void *vg, uint32_t desc) \
{ \
int i, oprsz = simd_oprsz(desc); \
for (i = 0; i < oprsz; ) { \
uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
do { \
if (pg & 1) { \
*(TYPE *)(vd + H(i)) = *(TYPE *)(za + tile_vslice_offset(i)); \
} \
i += sizeof(TYPE); \
pg >>= sizeof(TYPE); \
} while (i & 15); \
} \
}
DO_MOVA_Z(sme_mova_zc_b, uint8_t, H1)
DO_MOVA_Z(sme_mova_zc_h, uint16_t, H1_2)
DO_MOVA_Z(sme_mova_zc_s, uint32_t, H1_4)
void HELPER(sme_mova_zc_d)(void *vd, void *za, void *vg, uint32_t desc)
{
int i, oprsz = simd_oprsz(desc) / 8;
uint8_t *pg = vg;
uint64_t *d = vd;
uint64_t *a = za;
for (i = 0; i < oprsz; i++) {
if (pg[H1(i)] & 1) {
d[i] = a[tile_vslice_index(i)];
}
}
}
void HELPER(sme_mova_zc_q)(void *vd, void *za, void *vg, uint32_t desc)
{
int i, oprsz = simd_oprsz(desc) / 16;
uint16_t *pg = vg;
Int128 *d = vd;
Int128 *a = za;
/*
* Int128 is used here simply to copy 16 bytes, and to simplify
* the address arithmetic.
*/
for (i = 0; i < oprsz; i++, za += sizeof(ARMVectorReg)) {
if (pg[H2(i)] & 1) {
d[i] = a[tile_vslice_index(i)];
}
}
}
#undef DO_MOVA_Z

12
target/arm/sve_helper.c
View File

@ -3565,6 +3565,18 @@ void HELPER(sve_sel_zpzz_d)(void *vd, void *vn, void *vm,
}
}
void HELPER(sve_sel_zpzz_q)(void *vd, void *vn, void *vm,
void *vg, uint32_t desc)
{
intptr_t i, opr_sz = simd_oprsz(desc) / 16;
Int128 *d = vd, *n = vn, *m = vm;
uint16_t *pg = vg;
for (i = 0; i < opr_sz; i += 1) {
d[i] = (pg[H2(i)] & 1 ? n : m)[i];
}
}
/* Two operand comparison controlled by a predicate.
* ??? It is very tempting to want to be able to expand this inline
* with x86 instructions, e.g.

8
target/arm/translate-a64.h
View File

@ -178,6 +178,14 @@ static inline int pred_gvec_reg_size(DisasContext *s)
return size_for_gvec(pred_full_reg_size(s));
}
/* Return a newly allocated pointer to the predicate register. */
static inline TCGv_ptr pred_full_reg_ptr(DisasContext *s, int regno)
{
TCGv_ptr ret = tcg_temp_new_ptr();
tcg_gen_addi_ptr(ret, cpu_env, pred_full_reg_offset(s, regno));
return ret;
}
bool disas_sve(DisasContext *, uint32_t);
bool disas_sme(DisasContext *, uint32_t);

127
target/arm/translate-sme.c
View File

@ -35,6 +35,74 @@
#include "decode-sme.c.inc"
/*
* Resolve tile.size[index] to a host pointer, where tile and index
* are always decoded together, dependent on the element size.
*/
static TCGv_ptr get_tile_rowcol(DisasContext *s, int esz, int rs,
int tile_index, bool vertical)
{
int tile = tile_index >> (4 - esz);
int index = esz == MO_128 ? 0 : extract32(tile_index, 0, 4 - esz);
int pos, len, offset;
TCGv_i32 tmp;
TCGv_ptr addr;
/* Compute the final index, which is Rs+imm. */
tmp = tcg_temp_new_i32();
tcg_gen_trunc_tl_i32(tmp, cpu_reg(s, rs));
tcg_gen_addi_i32(tmp, tmp, index);
/* Prepare a power-of-two modulo via extraction of @len bits. */
len = ctz32(streaming_vec_reg_size(s)) - esz;
if (vertical) {
/*
* Compute the byte offset of the index within the tile:
* (index % (svl / size)) * size
* = (index % (svl >> esz)) << esz
* Perform the power-of-two modulo via extraction of the low @len bits.
* Perform the multiply by shifting left by @pos bits.
* Perform these operations simultaneously via deposit into zero.
*/
pos = esz;
tcg_gen_deposit_z_i32(tmp, tmp, pos, len);
/*
* For big-endian, adjust the indexed column byte offset within
* the uint64_t host words that make up env->zarray[].
*/
if (HOST_BIG_ENDIAN && esz < MO_64) {
tcg_gen_xori_i32(tmp, tmp, 8 - (1 << esz));
}
} else {
/*
* Compute the byte offset of the index within the tile:
* (index % (svl / size)) * (size * sizeof(row))
* = (index % (svl >> esz)) << (esz + log2(sizeof(row)))
*/
pos = esz + ctz32(sizeof(ARMVectorReg));
tcg_gen_deposit_z_i32(tmp, tmp, pos, len);
/* Row slices are always aligned and need no endian adjustment. */
}
/* The tile byte offset within env->zarray is the row. */
offset = tile * sizeof(ARMVectorReg);
/* Include the byte offset of zarray to make this relative to env. */
offset += offsetof(CPUARMState, zarray);
tcg_gen_addi_i32(tmp, tmp, offset);
/* Add the byte offset to env to produce the final pointer. */
addr = tcg_temp_new_ptr();
tcg_gen_ext_i32_ptr(addr, tmp);
tcg_temp_free_i32(tmp);
tcg_gen_add_ptr(addr, addr, cpu_env);
return addr;
}
static bool trans_ZERO(DisasContext *s, arg_ZERO *a)
{
if (!dc_isar_feature(aa64_sme, s)) {
@ -46,3 +114,62 @@ static bool trans_ZERO(DisasContext *s, arg_ZERO *a)
}
return true;
}
static bool trans_MOVA(DisasContext *s, arg_MOVA *a)
{
static gen_helper_gvec_4 * const h_fns[5] = {
gen_helper_sve_sel_zpzz_b, gen_helper_sve_sel_zpzz_h,
gen_helper_sve_sel_zpzz_s, gen_helper_sve_sel_zpzz_d,
gen_helper_sve_sel_zpzz_q
};
static gen_helper_gvec_3 * const cz_fns[5] = {
gen_helper_sme_mova_cz_b, gen_helper_sme_mova_cz_h,
gen_helper_sme_mova_cz_s, gen_helper_sme_mova_cz_d,
gen_helper_sme_mova_cz_q,
};
static gen_helper_gvec_3 * const zc_fns[5] = {
gen_helper_sme_mova_zc_b, gen_helper_sme_mova_zc_h,
gen_helper_sme_mova_zc_s, gen_helper_sme_mova_zc_d,
gen_helper_sme_mova_zc_q,
};
TCGv_ptr t_za, t_zr, t_pg;
TCGv_i32 t_desc;
int svl;
if (!dc_isar_feature(aa64_sme, s)) {
return false;
}
if (!sme_smza_enabled_check(s)) {
return true;
}
t_za = get_tile_rowcol(s, a->esz, a->rs, a->za_imm, a->v);
t_zr = vec_full_reg_ptr(s, a->zr);
t_pg = pred_full_reg_ptr(s, a->pg);
svl = streaming_vec_reg_size(s);
t_desc = tcg_constant_i32(simd_desc(svl, svl, 0));
if (a->v) {
/* Vertical slice -- use sme mova helpers. */
if (a->to_vec) {
zc_fns[a->esz](t_zr, t_za, t_pg, t_desc);
} else {
cz_fns[a->esz](t_za, t_zr, t_pg, t_desc);
}
} else {
/* Horizontal slice -- reuse sve sel helpers. */
if (a->to_vec) {
h_fns[a->esz](t_zr, t_za, t_zr, t_pg, t_desc);
} else {
h_fns[a->esz](t_za, t_zr, t_za, t_pg, t_desc);
}
}
tcg_temp_free_ptr(t_za);
tcg_temp_free_ptr(t_zr);
tcg_temp_free_ptr(t_pg);
return true;
}

5
target/arm/translate.h
View File

@ -156,6 +156,11 @@ static inline int plus_2(DisasContext *s, int x)
return x + 2;
}
static inline int plus_12(DisasContext *s, int x)
{
return x + 12;
}
static inline int times_2(DisasContext *s, int x)
{
return x * 2;