vect: Rename things related to rgroup_masks
Power supports vector memory access with length (in bytes) instructions. Like existing fully masking for SVE, it is another approach to vectorize the loop using partially-populated vectors. As Richard Sandiford pointed out, we can rename the rgroup struct rgroup_masks to rgroup_controls, rename its members mask_type to type, masks to controls to be more generic. Besides, this patch also renames some functions like vect_set_loop_mask to vect_set_loop_control, release_vec_loop_masks to release_vec_loop_controls, vect_set_loop_masks_directly to vect_set_loop_controls_directly. Bootstrapped/regtested on aarch64-linux-gnu. gcc/ChangeLog: * tree-vect-loop-manip.c (vect_set_loop_mask): Renamed to ... (vect_set_loop_control): ... this. (vect_maybe_permute_loop_masks): Rename rgroup_masks related things. (vect_set_loop_masks_directly): Renamed to ... (vect_set_loop_controls_directly): ... this. Also rename some variables with ctrl instead of mask. Rename vect_set_loop_mask to vect_set_loop_control. (vect_set_loop_condition_masked): Rename rgroup_masks related things. Also rename some variables with ctrl instead of mask. * tree-vect-loop.c (release_vec_loop_masks): Renamed to ... (release_vec_loop_controls): ... this. Rename rgroup_masks related things. (_loop_vec_info::~_loop_vec_info): Rename release_vec_loop_masks to release_vec_loop_controls. (can_produce_all_loop_masks_p): Rename rgroup_masks related things. (vect_get_max_nscalars_per_iter): Likewise. (vect_estimate_min_profitable_iters): Likewise. (vect_record_loop_mask): Likewise. (vect_get_loop_mask): Likewise. * tree-vectorizer.h (struct rgroup_masks): Renamed to ... (struct rgroup_controls): ... this. Also rename mask_type to type and rename masks to controls.
This commit is contained in:
Kewen Lin
Kewen Lin
parent
b3372425ec
commit
37478789dc
@ -256,17 +256,17 @@ adjust_phi_and_debug_stmts (gimple *update_phi, edge e, tree new_def)
|
||||
gimple_bb (update_phi));
|
||||
}
|
||||
|
||||
/* Define one loop mask MASK from loop LOOP. INIT_MASK is the value that
|
||||
the mask should have during the first iteration and NEXT_MASK is the
|
||||
/* Define one loop rgroup control CTRL from loop LOOP. INIT_CTRL is the value
|
||||
that the control should have during the first iteration and NEXT_CTRL is the
|
||||
value that it should have on subsequent iterations. */
|
||||
|
||||
static void
|
||||
vect_set_loop_mask (class loop *loop, tree mask, tree init_mask,
|
||||
tree next_mask)
|
||||
vect_set_loop_control (class loop *loop, tree ctrl, tree init_ctrl,
|
||||
tree next_ctrl)
|
||||
{
|
||||
gphi *phi = create_phi_node (mask, loop->header);
|
||||
add_phi_arg (phi, init_mask, loop_preheader_edge (loop), UNKNOWN_LOCATION);
|
||||
add_phi_arg (phi, next_mask, loop_latch_edge (loop), UNKNOWN_LOCATION);
|
||||
gphi *phi = create_phi_node (ctrl, loop->header);
|
||||
add_phi_arg (phi, init_ctrl, loop_preheader_edge (loop), UNKNOWN_LOCATION);
|
||||
add_phi_arg (phi, next_ctrl, loop_latch_edge (loop), UNKNOWN_LOCATION);
|
||||
}
|
||||
|
||||
/* Add SEQ to the end of LOOP's preheader block. */
|
||||
@ -320,11 +320,11 @@ interleave_supported_p (vec_perm_indices *indices, tree vectype,
|
||||
latter. Return true on success, adding any new statements to SEQ. */
|
||||
|
||||
static bool
|
||||
vect_maybe_permute_loop_masks (gimple_seq *seq, rgroup_masks *dest_rgm,
|
||||
rgroup_masks *src_rgm)
|
||||
vect_maybe_permute_loop_masks (gimple_seq *seq, rgroup_controls *dest_rgm,
|
||||
rgroup_controls *src_rgm)
|
||||
{
|
||||
tree src_masktype = src_rgm->mask_type;
|
||||
tree dest_masktype = dest_rgm->mask_type;
|
||||
tree src_masktype = src_rgm->type;
|
||||
tree dest_masktype = dest_rgm->type;
|
||||
machine_mode src_mode = TYPE_MODE (src_masktype);
|
||||
insn_code icode1, icode2;
|
||||
if (dest_rgm->max_nscalars_per_iter <= src_rgm->max_nscalars_per_iter
|
||||
@ -338,10 +338,10 @@ vect_maybe_permute_loop_masks (gimple_seq *seq, rgroup_masks *dest_rgm,
|
||||
machine_mode dest_mode = insn_data[icode1].operand[0].mode;
|
||||
gcc_assert (dest_mode == insn_data[icode2].operand[0].mode);
|
||||
tree unpack_masktype = vect_halve_mask_nunits (src_masktype, dest_mode);
|
||||
for (unsigned int i = 0; i < dest_rgm->masks.length (); ++i)
|
||||
for (unsigned int i = 0; i < dest_rgm->controls.length (); ++i)
|
||||
{
|
||||
tree src = src_rgm->masks[i / 2];
|
||||
tree dest = dest_rgm->masks[i];
|
||||
tree src = src_rgm->controls[i / 2];
|
||||
tree dest = dest_rgm->controls[i];
|
||||
tree_code code = ((i & 1) == (BYTES_BIG_ENDIAN ? 0 : 1)
|
||||
? VEC_UNPACK_HI_EXPR
|
||||
: VEC_UNPACK_LO_EXPR);
|
||||
@ -371,10 +371,10 @@ vect_maybe_permute_loop_masks (gimple_seq *seq, rgroup_masks *dest_rgm,
|
||||
tree masks[2];
|
||||
for (unsigned int i = 0; i < 2; ++i)
|
||||
masks[i] = vect_gen_perm_mask_checked (src_masktype, indices[i]);
|
||||
for (unsigned int i = 0; i < dest_rgm->masks.length (); ++i)
|
||||
for (unsigned int i = 0; i < dest_rgm->controls.length (); ++i)
|
||||
{
|
||||
tree src = src_rgm->masks[i / 2];
|
||||
tree dest = dest_rgm->masks[i];
|
||||
tree src = src_rgm->controls[i / 2];
|
||||
tree dest = dest_rgm->controls[i];
|
||||
gimple *stmt = gimple_build_assign (dest, VEC_PERM_EXPR,
|
||||
src, src, masks[i & 1]);
|
||||
gimple_seq_add_stmt (seq, stmt);
|
||||
@ -384,12 +384,12 @@ vect_maybe_permute_loop_masks (gimple_seq *seq, rgroup_masks *dest_rgm,
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Helper for vect_set_loop_condition_masked. Generate definitions for
|
||||
all the masks in RGM and return a mask that is nonzero when the loop
|
||||
needs to iterate. Add any new preheader statements to PREHEADER_SEQ.
|
||||
Use LOOP_COND_GSI to insert code before the exit gcond.
|
||||
/* Helper for vect_set_loop_condition_partial_vectors. Generate definitions
|
||||
for all the rgroup controls in RGC and return a control that is nonzero
|
||||
when the loop needs to iterate. Add any new preheader statements to
|
||||
PREHEADER_SEQ. Use LOOP_COND_GSI to insert code before the exit gcond.
|
||||
|
||||
RGM belongs to loop LOOP. The loop originally iterated NITERS
|
||||
RGC belongs to loop LOOP. The loop originally iterated NITERS
|
||||
times and has been vectorized according to LOOP_VINFO.
|
||||
|
||||
If NITERS_SKIP is nonnull, the first iteration of the vectorized loop
|
||||
@ -399,32 +399,32 @@ vect_maybe_permute_loop_masks (gimple_seq *seq, rgroup_masks *dest_rgm,
|
||||
|
||||
It is known that:
|
||||
|
||||
NITERS * RGM->max_nscalars_per_iter
|
||||
NITERS * RGC->max_nscalars_per_iter
|
||||
|
||||
does not overflow. However, MIGHT_WRAP_P says whether an induction
|
||||
variable that starts at 0 and has step:
|
||||
|
||||
VF * RGM->max_nscalars_per_iter
|
||||
VF * RGC->max_nscalars_per_iter
|
||||
|
||||
might overflow before hitting a value above:
|
||||
|
||||
(NITERS + NITERS_SKIP) * RGM->max_nscalars_per_iter
|
||||
(NITERS + NITERS_SKIP) * RGC->max_nscalars_per_iter
|
||||
|
||||
This means that we cannot guarantee that such an induction variable
|
||||
would ever hit a value that produces a set of all-false masks for RGM. */
|
||||
would ever hit a value that produces a set of all-false masks for RGC. */
|
||||
|
||||
static tree
|
||||
vect_set_loop_masks_directly (class loop *loop, loop_vec_info loop_vinfo,
|
||||
gimple_seq *preheader_seq,
|
||||
gimple_stmt_iterator loop_cond_gsi,
|
||||
rgroup_masks *rgm, tree niters, tree niters_skip,
|
||||
bool might_wrap_p)
|
||||
vect_set_loop_controls_directly (class loop *loop, loop_vec_info loop_vinfo,
|
||||
gimple_seq *preheader_seq,
|
||||
gimple_stmt_iterator loop_cond_gsi,
|
||||
rgroup_controls *rgc, tree niters,
|
||||
tree niters_skip, bool might_wrap_p)
|
||||
{
|
||||
tree compare_type = LOOP_VINFO_MASK_COMPARE_TYPE (loop_vinfo);
|
||||
tree iv_type = LOOP_VINFO_MASK_IV_TYPE (loop_vinfo);
|
||||
tree mask_type = rgm->mask_type;
|
||||
unsigned int nscalars_per_iter = rgm->max_nscalars_per_iter;
|
||||
poly_uint64 nscalars_per_mask = TYPE_VECTOR_SUBPARTS (mask_type);
|
||||
tree ctrl_type = rgc->type;
|
||||
unsigned int nscalars_per_iter = rgc->max_nscalars_per_iter;
|
||||
poly_uint64 nscalars_per_ctrl = TYPE_VECTOR_SUBPARTS (ctrl_type);
|
||||
poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
|
||||
|
||||
/* Calculate the maximum number of scalar values that the rgroup
|
||||
@ -436,8 +436,8 @@ vect_set_loop_masks_directly (class loop *loop, loop_vec_info loop_vinfo,
|
||||
tree nscalars_skip = niters_skip;
|
||||
if (nscalars_per_iter != 1)
|
||||
{
|
||||
/* We checked before choosing to use a fully-masked loop that these
|
||||
multiplications don't overflow. */
|
||||
/* We checked before setting LOOP_VINFO_USING_PARTIAL_VECTORS_P that
|
||||
these multiplications don't overflow. */
|
||||
tree compare_factor = build_int_cst (compare_type, nscalars_per_iter);
|
||||
tree iv_factor = build_int_cst (iv_type, nscalars_per_iter);
|
||||
nscalars_total = gimple_build (preheader_seq, MULT_EXPR, compare_type,
|
||||
@ -541,27 +541,27 @@ vect_set_loop_masks_directly (class loop *loop, loop_vec_info loop_vinfo,
|
||||
test_index = gimple_convert (&test_seq, compare_type, test_index);
|
||||
gsi_insert_seq_before (test_gsi, test_seq, GSI_SAME_STMT);
|
||||
|
||||
/* Provide a definition of each mask in the group. */
|
||||
tree next_mask = NULL_TREE;
|
||||
tree mask;
|
||||
/* Provide a definition of each control in the group. */
|
||||
tree next_ctrl = NULL_TREE;
|
||||
tree ctrl;
|
||||
unsigned int i;
|
||||
FOR_EACH_VEC_ELT_REVERSE (rgm->masks, i, mask)
|
||||
FOR_EACH_VEC_ELT_REVERSE (rgc->controls, i, ctrl)
|
||||
{
|
||||
/* Previous masks will cover BIAS scalars. This mask covers the
|
||||
/* Previous controls will cover BIAS scalars. This control covers the
|
||||
next batch. */
|
||||
poly_uint64 bias = nscalars_per_mask * i;
|
||||
poly_uint64 bias = nscalars_per_ctrl * i;
|
||||
tree bias_tree = build_int_cst (compare_type, bias);
|
||||
gimple *tmp_stmt;
|
||||
|
||||
/* See whether the first iteration of the vector loop is known
|
||||
to have a full mask. */
|
||||
to have a full control. */
|
||||
poly_uint64 const_limit;
|
||||
bool first_iteration_full
|
||||
= (poly_int_tree_p (first_limit, &const_limit)
|
||||
&& known_ge (const_limit, (i + 1) * nscalars_per_mask));
|
||||
&& known_ge (const_limit, (i + 1) * nscalars_per_ctrl));
|
||||
|
||||
/* Rather than have a new IV that starts at BIAS and goes up to
|
||||
TEST_LIMIT, prefer to use the same 0-based IV for each mask
|
||||
TEST_LIMIT, prefer to use the same 0-based IV for each control
|
||||
and adjust the bound down by BIAS. */
|
||||
tree this_test_limit = test_limit;
|
||||
if (i != 0)
|
||||
@ -574,9 +574,9 @@ vect_set_loop_masks_directly (class loop *loop, loop_vec_info loop_vinfo,
|
||||
bias_tree);
|
||||
}
|
||||
|
||||
/* Create the initial mask. First include all scalars that
|
||||
/* Create the initial control. First include all scalars that
|
||||
are within the loop limit. */
|
||||
tree init_mask = NULL_TREE;
|
||||
tree init_ctrl = NULL_TREE;
|
||||
if (!first_iteration_full)
|
||||
{
|
||||
tree start, end;
|
||||
@ -593,13 +593,13 @@ vect_set_loop_masks_directly (class loop *loop, loop_vec_info loop_vinfo,
|
||||
{
|
||||
/* FIRST_LIMIT is the maximum number of scalars handled by the
|
||||
first iteration of the vector loop. Test the portion
|
||||
associated with this mask. */
|
||||
associated with this control. */
|
||||
start = bias_tree;
|
||||
end = first_limit;
|
||||
}
|
||||
|
||||
init_mask = make_temp_ssa_name (mask_type, NULL, "max_mask");
|
||||
tmp_stmt = vect_gen_while (init_mask, start, end);
|
||||
init_ctrl = make_temp_ssa_name (ctrl_type, NULL, "max_mask");
|
||||
tmp_stmt = vect_gen_while (init_ctrl, start, end);
|
||||
gimple_seq_add_stmt (preheader_seq, tmp_stmt);
|
||||
}
|
||||
|
||||
@ -610,34 +610,35 @@ vect_set_loop_masks_directly (class loop *loop, loop_vec_info loop_vinfo,
|
||||
&& !(poly_int_tree_p (nscalars_skip, &const_skip)
|
||||
&& known_le (const_skip, bias)))
|
||||
{
|
||||
tree unskipped_mask = vect_gen_while_not (preheader_seq, mask_type,
|
||||
tree unskipped_mask = vect_gen_while_not (preheader_seq, ctrl_type,
|
||||
bias_tree, nscalars_skip);
|
||||
if (init_mask)
|
||||
init_mask = gimple_build (preheader_seq, BIT_AND_EXPR, mask_type,
|
||||
init_mask, unskipped_mask);
|
||||
if (init_ctrl)
|
||||
init_ctrl = gimple_build (preheader_seq, BIT_AND_EXPR, ctrl_type,
|
||||
init_ctrl, unskipped_mask);
|
||||
else
|
||||
init_mask = unskipped_mask;
|
||||
init_ctrl = unskipped_mask;
|
||||
}
|
||||
|
||||
if (!init_mask)
|
||||
if (!init_ctrl)
|
||||
/* First iteration is full. */
|
||||
init_mask = build_minus_one_cst (mask_type);
|
||||
init_ctrl = build_minus_one_cst (ctrl_type);
|
||||
|
||||
/* Get the mask value for the next iteration of the loop. */
|
||||
next_mask = make_temp_ssa_name (mask_type, NULL, "next_mask");
|
||||
gcall *call = vect_gen_while (next_mask, test_index, this_test_limit);
|
||||
/* Get the control value for the next iteration of the loop. */
|
||||
next_ctrl = make_temp_ssa_name (ctrl_type, NULL, "next_mask");
|
||||
gcall *call = vect_gen_while (next_ctrl, test_index, this_test_limit);
|
||||
gsi_insert_before (test_gsi, call, GSI_SAME_STMT);
|
||||
|
||||
vect_set_loop_mask (loop, mask, init_mask, next_mask);
|
||||
vect_set_loop_control (loop, ctrl, init_ctrl, next_ctrl);
|
||||
}
|
||||
return next_mask;
|
||||
return next_ctrl;
|
||||
}
|
||||
|
||||
/* Make LOOP iterate NITERS times using masking and WHILE_ULT calls.
|
||||
LOOP_VINFO describes the vectorization of LOOP. NITERS is the
|
||||
number of iterations of the original scalar loop that should be
|
||||
handled by the vector loop. NITERS_MAYBE_ZERO and FINAL_IV are
|
||||
as for vect_set_loop_condition.
|
||||
/* Set up the iteration condition and rgroup controls for LOOP, given
|
||||
that LOOP_VINFO_USING_PARTIAL_VECTORS_P is true for the vectorized
|
||||
loop. LOOP_VINFO describes the vectorization of LOOP. NITERS is
|
||||
the number of iterations of the original scalar loop that should be
|
||||
handled by the vector loop. NITERS_MAYBE_ZERO and FINAL_IV are as
|
||||
for vect_set_loop_condition.
|
||||
|
||||
Insert the branch-back condition before LOOP_COND_GSI and return the
|
||||
final gcond. */
|
||||
@ -679,15 +680,15 @@ vect_set_loop_condition_masked (class loop *loop, loop_vec_info loop_vinfo,
|
||||
|
||||
widest_int iv_limit = vect_iv_limit_for_full_masking (loop_vinfo);
|
||||
|
||||
/* Iterate over all the rgroups and fill in their masks. We could use
|
||||
the first mask from any rgroup for the loop condition; here we
|
||||
/* Iterate over all the rgroups and fill in their controls. We could use
|
||||
the first control from any rgroup for the loop condition; here we
|
||||
arbitrarily pick the last. */
|
||||
tree test_mask = NULL_TREE;
|
||||
rgroup_masks *rgm;
|
||||
tree test_ctrl = NULL_TREE;
|
||||
rgroup_controls *rgc;
|
||||
unsigned int i;
|
||||
vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
|
||||
FOR_EACH_VEC_ELT (*masks, i, rgm)
|
||||
if (!rgm->masks.is_empty ())
|
||||
FOR_EACH_VEC_ELT (*masks, i, rgc)
|
||||
if (!rgc->controls.is_empty ())
|
||||
{
|
||||
/* First try using permutes. This adds a single vector
|
||||
instruction to the loop for each mask, but needs no extra
|
||||
@ -695,9 +696,9 @@ vect_set_loop_condition_masked (class loop *loop, loop_vec_info loop_vinfo,
|
||||
unsigned int nmasks = i + 1;
|
||||
if ((nmasks & 1) == 0)
|
||||
{
|
||||
rgroup_masks *half_rgm = &(*masks)[nmasks / 2 - 1];
|
||||
if (!half_rgm->masks.is_empty ()
|
||||
&& vect_maybe_permute_loop_masks (&header_seq, rgm, half_rgm))
|
||||
rgroup_controls *half_rgc = &(*masks)[nmasks / 2 - 1];
|
||||
if (!half_rgc->controls.is_empty ()
|
||||
&& vect_maybe_permute_loop_masks (&header_seq, rgc, half_rgc))
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -705,16 +706,16 @@ vect_set_loop_condition_masked (class loop *loop, loop_vec_info loop_vinfo,
|
||||
before wrapping around. */
|
||||
bool might_wrap_p
|
||||
= (iv_limit == -1
|
||||
|| (wi::min_precision (iv_limit * rgm->max_nscalars_per_iter,
|
||||
|| (wi::min_precision (iv_limit * rgc->max_nscalars_per_iter,
|
||||
UNSIGNED)
|
||||
> compare_precision));
|
||||
|
||||
/* Set up all masks for this group. */
|
||||
test_mask = vect_set_loop_masks_directly (loop, loop_vinfo,
|
||||
&preheader_seq,
|
||||
loop_cond_gsi, rgm,
|
||||
niters, niters_skip,
|
||||
might_wrap_p);
|
||||
/* Set up all controls for this group. */
|
||||
test_ctrl = vect_set_loop_controls_directly (loop, loop_vinfo,
|
||||
&preheader_seq,
|
||||
loop_cond_gsi, rgc,
|
||||
niters, niters_skip,
|
||||
might_wrap_p);
|
||||
}
|
||||
|
||||
/* Emit all accumulated statements. */
|
||||
@ -724,8 +725,8 @@ vect_set_loop_condition_masked (class loop *loop, loop_vec_info loop_vinfo,
|
||||
/* Get a boolean result that tells us whether to iterate. */
|
||||
edge exit_edge = single_exit (loop);
|
||||
tree_code code = (exit_edge->flags & EDGE_TRUE_VALUE) ? EQ_EXPR : NE_EXPR;
|
||||
tree zero_mask = build_zero_cst (TREE_TYPE (test_mask));
|
||||
gcond *cond_stmt = gimple_build_cond (code, test_mask, zero_mask,
|
||||
tree zero_ctrl = build_zero_cst (TREE_TYPE (test_ctrl));
|
||||
gcond *cond_stmt = gimple_build_cond (code, test_ctrl, zero_ctrl,
|
||||
NULL_TREE, NULL_TREE);
|
||||
gsi_insert_before (&loop_cond_gsi, cond_stmt, GSI_SAME_STMT);
|
||||
|
||||
|
||||
@ -878,16 +878,16 @@ _loop_vec_info::_loop_vec_info (class loop *loop_in, vec_info_shared *shared)
|
||||
epilogue_vinfos.create (6);
|
||||
}
|
||||
|
||||
/* Free all levels of MASKS. */
|
||||
/* Free all levels of rgroup CONTROLS. */
|
||||
|
||||
void
|
||||
release_vec_loop_masks (vec_loop_masks *masks)
|
||||
release_vec_loop_controls (vec<rgroup_controls> *controls)
|
||||
{
|
||||
rgroup_masks *rgm;
|
||||
rgroup_controls *rgc;
|
||||
unsigned int i;
|
||||
FOR_EACH_VEC_ELT (*masks, i, rgm)
|
||||
rgm->masks.release ();
|
||||
masks->release ();
|
||||
FOR_EACH_VEC_ELT (*controls, i, rgc)
|
||||
rgc->controls.release ();
|
||||
controls->release ();
|
||||
}
|
||||
|
||||
/* Free all memory used by the _loop_vec_info, as well as all the
|
||||
@ -897,7 +897,7 @@ _loop_vec_info::~_loop_vec_info ()
|
||||
{
|
||||
free (bbs);
|
||||
|
||||
release_vec_loop_masks (&masks);
|
||||
release_vec_loop_controls (&masks);
|
||||
delete ivexpr_map;
|
||||
delete scan_map;
|
||||
epilogue_vinfos.release ();
|
||||
@ -938,12 +938,12 @@ cse_and_gimplify_to_preheader (loop_vec_info loop_vinfo, tree expr)
|
||||
static bool
|
||||
can_produce_all_loop_masks_p (loop_vec_info loop_vinfo, tree cmp_type)
|
||||
{
|
||||
rgroup_masks *rgm;
|
||||
rgroup_controls *rgm;
|
||||
unsigned int i;
|
||||
FOR_EACH_VEC_ELT (LOOP_VINFO_MASKS (loop_vinfo), i, rgm)
|
||||
if (rgm->mask_type != NULL_TREE
|
||||
if (rgm->type != NULL_TREE
|
||||
&& !direct_internal_fn_supported_p (IFN_WHILE_ULT,
|
||||
cmp_type, rgm->mask_type,
|
||||
cmp_type, rgm->type,
|
||||
OPTIMIZE_FOR_SPEED))
|
||||
return false;
|
||||
return true;
|
||||
@ -957,7 +957,7 @@ vect_get_max_nscalars_per_iter (loop_vec_info loop_vinfo)
|
||||
{
|
||||
unsigned int res = 1;
|
||||
unsigned int i;
|
||||
rgroup_masks *rgm;
|
||||
rgroup_controls *rgm;
|
||||
FOR_EACH_VEC_ELT (LOOP_VINFO_MASKS (loop_vinfo), i, rgm)
|
||||
res = MAX (res, rgm->max_nscalars_per_iter);
|
||||
return res;
|
||||
@ -2379,7 +2379,7 @@ again:
|
||||
LOOP_VINFO_TARGET_COST_DATA (loop_vinfo)
|
||||
= init_cost (LOOP_VINFO_LOOP (loop_vinfo));
|
||||
/* Reset accumulated rgroup information. */
|
||||
release_vec_loop_masks (&LOOP_VINFO_MASKS (loop_vinfo));
|
||||
release_vec_loop_controls (&LOOP_VINFO_MASKS (loop_vinfo));
|
||||
/* Reset assorted flags. */
|
||||
LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo) = false;
|
||||
LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) = false;
|
||||
@ -3512,10 +3512,10 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
|
||||
|
||||
/* Calculate how many masks we need to generate. */
|
||||
unsigned int num_masks = 0;
|
||||
rgroup_masks *rgm;
|
||||
rgroup_controls *rgm;
|
||||
unsigned int num_vectors_m1;
|
||||
FOR_EACH_VEC_ELT (LOOP_VINFO_MASKS (loop_vinfo), num_vectors_m1, rgm)
|
||||
if (rgm->mask_type)
|
||||
if (rgm->type)
|
||||
num_masks += num_vectors_m1 + 1;
|
||||
gcc_assert (num_masks > 0);
|
||||
|
||||
@ -8153,7 +8153,7 @@ vect_record_loop_mask (loop_vec_info loop_vinfo, vec_loop_masks *masks,
|
||||
gcc_assert (nvectors != 0);
|
||||
if (masks->length () < nvectors)
|
||||
masks->safe_grow_cleared (nvectors);
|
||||
rgroup_masks *rgm = &(*masks)[nvectors - 1];
|
||||
rgroup_controls *rgm = &(*masks)[nvectors - 1];
|
||||
/* The number of scalars per iteration and the number of vectors are
|
||||
both compile-time constants. */
|
||||
unsigned int nscalars_per_iter
|
||||
@ -8169,7 +8169,7 @@ vect_record_loop_mask (loop_vec_info loop_vinfo, vec_loop_masks *masks,
|
||||
if (rgm->max_nscalars_per_iter < nscalars_per_iter)
|
||||
{
|
||||
rgm->max_nscalars_per_iter = nscalars_per_iter;
|
||||
rgm->mask_type = truth_type_for (vectype);
|
||||
rgm->type = truth_type_for (vectype);
|
||||
}
|
||||
}
|
||||
|
||||
@ -8184,24 +8184,24 @@ tree
|
||||
vect_get_loop_mask (gimple_stmt_iterator *gsi, vec_loop_masks *masks,
|
||||
unsigned int nvectors, tree vectype, unsigned int index)
|
||||
{
|
||||
rgroup_masks *rgm = &(*masks)[nvectors - 1];
|
||||
tree mask_type = rgm->mask_type;
|
||||
rgroup_controls *rgm = &(*masks)[nvectors - 1];
|
||||
tree mask_type = rgm->type;
|
||||
|
||||
/* Populate the rgroup's mask array, if this is the first time we've
|
||||
used it. */
|
||||
if (rgm->masks.is_empty ())
|
||||
if (rgm->controls.is_empty ())
|
||||
{
|
||||
rgm->masks.safe_grow_cleared (nvectors);
|
||||
rgm->controls.safe_grow_cleared (nvectors);
|
||||
for (unsigned int i = 0; i < nvectors; ++i)
|
||||
{
|
||||
tree mask = make_temp_ssa_name (mask_type, NULL, "loop_mask");
|
||||
/* Provide a dummy definition until the real one is available. */
|
||||
SSA_NAME_DEF_STMT (mask) = gimple_build_nop ();
|
||||
rgm->masks[i] = mask;
|
||||
rgm->controls[i] = mask;
|
||||
}
|
||||
}
|
||||
|
||||
tree mask = rgm->masks[index];
|
||||
tree mask = rgm->controls[index];
|
||||
if (maybe_ne (TYPE_VECTOR_SUBPARTS (mask_type),
|
||||
TYPE_VECTOR_SUBPARTS (vectype)))
|
||||
{
|
||||
|
||||
@ -390,7 +390,6 @@ is_a_helper <_bb_vec_info *>::test (vec_info *i)
|
||||
return i->kind == vec_info::bb;
|
||||
}
|
||||
|
||||
|
||||
/* In general, we can divide the vector statements in a vectorized loop
|
||||
into related groups ("rgroups") and say that for each rgroup there is
|
||||
some nS such that the rgroup operates on nS values from one scalar
|
||||
@ -420,19 +419,21 @@ is_a_helper <_bb_vec_info *>::test (vec_info *i)
|
||||
|
||||
In classical vectorization, each iteration of the vector loop would
|
||||
handle exactly VF iterations of the original scalar loop. However,
|
||||
in a fully-masked loop, a particular iteration of the vector loop
|
||||
might handle fewer than VF iterations of the scalar loop. The vector
|
||||
lanes that correspond to iterations of the scalar loop are said to be
|
||||
"active" and the other lanes are said to be "inactive".
|
||||
in vector loops that are able to operate on partial vectors, a
|
||||
particular iteration of the vector loop might handle fewer than VF
|
||||
iterations of the scalar loop. The vector lanes that correspond to
|
||||
iterations of the scalar loop are said to be "active" and the other
|
||||
lanes are said to be "inactive".
|
||||
|
||||
In a fully-masked loop, many rgroups need to be masked to ensure that
|
||||
they have no effect for the inactive lanes. Each such rgroup needs a
|
||||
sequence of booleans in the same order as above, but with each (i,j)
|
||||
replaced by a boolean that indicates whether iteration i is active.
|
||||
This sequence occupies nV vector masks that again have nL lanes each.
|
||||
Thus the mask sequence as a whole consists of VF independent booleans
|
||||
that are each repeated nS times.
|
||||
In such vector loops, many rgroups need to be controlled to ensure
|
||||
that they have no effect for the inactive lanes. Conceptually, each
|
||||
such rgroup needs a sequence of booleans in the same order as above,
|
||||
but with each (i,j) replaced by a boolean that indicates whether
|
||||
iteration i is active. This sequence occupies nV vector controls
|
||||
that again have nL lanes each. Thus the control sequence as a whole
|
||||
consists of VF independent booleans that are each repeated nS times.
|
||||
|
||||
Taking mask-based approach as a partially-populated vectors example.
|
||||
We make the simplifying assumption that if a sequence of nV masks is
|
||||
suitable for one (nS,nL) pair, we can reuse it for (nS/2,nL/2) by
|
||||
VIEW_CONVERTing it. This holds for all current targets that support
|
||||
@ -472,20 +473,21 @@ is_a_helper <_bb_vec_info *>::test (vec_info *i)
|
||||
first level being indexed by nV - 1 (since nV == 0 doesn't exist) and
|
||||
the second being indexed by the mask index 0 <= i < nV. */
|
||||
|
||||
/* The masks needed by rgroups with nV vectors, according to the
|
||||
description above. */
|
||||
struct rgroup_masks {
|
||||
/* The largest nS for all rgroups that use these masks. */
|
||||
/* The controls (like masks) needed by rgroups with nV vectors,
|
||||
according to the description above. */
|
||||
struct rgroup_controls {
|
||||
/* The largest nS for all rgroups that use these controls. */
|
||||
unsigned int max_nscalars_per_iter;
|
||||
|
||||
/* The type of mask to use, based on the highest nS recorded above. */
|
||||
tree mask_type;
|
||||
/* The type of control to use, based on the highest nS recorded above.
|
||||
For mask-based approach, it's used for mask_type. */
|
||||
tree type;
|
||||
|
||||
/* A vector of nV masks, in iteration order. */
|
||||
vec<tree> masks;
|
||||
/* A vector of nV controls, in iteration order. */
|
||||
vec<tree> controls;
|
||||
};
|
||||
|
||||
typedef auto_vec<rgroup_masks> vec_loop_masks;
|
||||
typedef auto_vec<rgroup_controls> vec_loop_masks;
|
||||
|
||||
typedef auto_vec<std::pair<data_reference*, tree> > drs_init_vec;
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user