gimple.c (gimple_get_lhs): For PHIs return the result.
2019-09-30 Richard Biener <rguenther@suse.de> * gimple.c (gimple_get_lhs): For PHIs return the result. * tree-vectorizer.h (vectorizable_live_operation): Also get the SLP instance as argument. * tree-vect-loop.c (vect_analyze_loop_operations): Also handle double-reduction PHIs with vectorizable_lc_phi. (vect_analyze_loop_operations): Adjust. (vect_create_epilog_for_reduction): Remove all code not dealing with reduction LC PHI or epilogue generation. (vectorizable_live_operation): Call vect_create_epilog_for_reduction for live stmts of reductions. * tree-vect-stmts.c (vectorizable_condition): When !for_reduction do not handle defs that are not vect_internal_def. (can_vectorize_live_stmts): Adjust. (vect_analyze_stmt): When the vectorized stmt defined a value used on backedges adjust the backedge uses of vectorized PHIs. From-SVN: r276299
This commit is contained in:
parent
582c57a17e
commit
61362d9d18
|
@ -1,3 +1,21 @@
|
|||
2019-09-30 Richard Biener <rguenther@suse.de>
|
||||
|
||||
* gimple.c (gimple_get_lhs): For PHIs return the result.
|
||||
* tree-vectorizer.h (vectorizable_live_operation): Also get the
|
||||
SLP instance as argument.
|
||||
* tree-vect-loop.c (vect_analyze_loop_operations): Also handle
|
||||
double-reduction PHIs with vectorizable_lc_phi.
|
||||
(vect_analyze_loop_operations): Adjust.
|
||||
(vect_create_epilog_for_reduction): Remove all code not dealing
|
||||
with reduction LC PHI or epilogue generation.
|
||||
(vectorizable_live_operation): Call vect_create_epilog_for_reduction
|
||||
for live stmts of reductions.
|
||||
* tree-vect-stmts.c (vectorizable_condition): When !for_reduction
|
||||
do not handle defs that are not vect_internal_def.
|
||||
(can_vectorize_live_stmts): Adjust.
|
||||
(vect_analyze_stmt): When the vectorized stmt defined a value
|
||||
used on backedges adjust the backedge uses of vectorized PHIs.
|
||||
|
||||
2019-09-30 Martin Jambor <mjambor@suse.cz>
|
||||
|
||||
PR ipa/91853
|
||||
|
|
|
@ -1797,6 +1797,8 @@ gimple_get_lhs (const gimple *stmt)
|
|||
return gimple_assign_lhs (stmt);
|
||||
else if (code == GIMPLE_CALL)
|
||||
return gimple_call_lhs (stmt);
|
||||
else if (code == GIMPLE_PHI)
|
||||
return gimple_phi_result (stmt);
|
||||
else
|
||||
return NULL_TREE;
|
||||
}
|
||||
|
|
|
@ -1526,7 +1526,9 @@ vect_analyze_loop_operations (loop_vec_info loop_vinfo)
|
|||
!= vect_used_in_outer_by_reduction))
|
||||
return opt_result::failure_at (phi, "unsupported phi\n");
|
||||
|
||||
if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_internal_def
|
||||
if ((STMT_VINFO_DEF_TYPE (stmt_info) == vect_internal_def
|
||||
|| (STMT_VINFO_DEF_TYPE (stmt_info)
|
||||
== vect_double_reduction_def))
|
||||
&& !vectorizable_lc_phi (stmt_info, NULL, NULL))
|
||||
return opt_result::failure_at (phi, "unsupported phi\n");
|
||||
}
|
||||
|
@ -1564,8 +1566,8 @@ vect_analyze_loop_operations (loop_vec_info loop_vinfo)
|
|||
if (ok
|
||||
&& STMT_VINFO_LIVE_P (stmt_info)
|
||||
&& !PURE_SLP_STMT (stmt_info))
|
||||
ok = vectorizable_live_operation (stmt_info, NULL, NULL, -1, NULL,
|
||||
&cost_vec);
|
||||
ok = vectorizable_live_operation (stmt_info, NULL, NULL, NULL,
|
||||
-1, NULL, &cost_vec);
|
||||
|
||||
if (!ok)
|
||||
return opt_result::failure_at (phi,
|
||||
|
@ -4260,6 +4262,20 @@ vect_create_epilog_for_reduction (stmt_vec_info stmt_info,
|
|||
slp_instance slp_node_instance)
|
||||
{
|
||||
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
|
||||
/* For double reductions we need to get at the inner loop reduction
|
||||
stmt which has the meta info attached. Our stmt_info is that of the
|
||||
loop-closed PHI of the inner loop which we remember as
|
||||
def for the reduction PHI generation. */
|
||||
bool double_reduc = false;
|
||||
stmt_vec_info rdef_info = stmt_info;
|
||||
if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
|
||||
{
|
||||
gcc_assert (!slp_node);
|
||||
double_reduc = true;
|
||||
stmt_info = loop_vinfo->lookup_def (gimple_phi_arg_def
|
||||
(stmt_info->stmt, 0));
|
||||
stmt_info = vect_stmt_to_vectorize (stmt_info);
|
||||
}
|
||||
gphi *reduc_def_stmt
|
||||
= as_a <gphi *> (STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info))->stmt);
|
||||
enum tree_code code = STMT_VINFO_REDUC_CODE (stmt_info);
|
||||
|
@ -4269,20 +4285,6 @@ vect_create_epilog_for_reduction (stmt_vec_info stmt_info,
|
|||
neutral_op
|
||||
= neutral_op_for_slp_reduction (slp_node_instance->reduc_phis, code,
|
||||
REDUC_GROUP_FIRST_ELEMENT (stmt_info));
|
||||
bool double_reduc = false;
|
||||
if (nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt_info))
|
||||
{
|
||||
tree def_arg
|
||||
= PHI_ARG_DEF_FROM_EDGE (reduc_def_stmt,
|
||||
loop_preheader_edge
|
||||
(LOOP_VINFO_LOOP (loop_vinfo)->inner));
|
||||
stmt_vec_info def_arg_stmt_info = loop_vinfo->lookup_def (def_arg);
|
||||
if (def_arg_stmt_info
|
||||
&& (STMT_VINFO_DEF_TYPE (def_arg_stmt_info)
|
||||
== vect_double_reduction_def))
|
||||
double_reduc = true;
|
||||
}
|
||||
tree induc_val = NULL_TREE;
|
||||
stmt_vec_info prev_phi_info;
|
||||
tree vectype;
|
||||
machine_mode mode;
|
||||
|
@ -4298,23 +4300,20 @@ vect_create_epilog_for_reduction (stmt_vec_info stmt_info,
|
|||
gimple *epilog_stmt = NULL;
|
||||
gimple *exit_phi;
|
||||
tree bitsize;
|
||||
tree adjustment_def = NULL;
|
||||
tree expr, def, initial_def = NULL;
|
||||
tree expr, def;
|
||||
tree orig_name, scalar_result;
|
||||
imm_use_iterator imm_iter, phi_imm_iter;
|
||||
use_operand_p use_p, phi_use_p;
|
||||
gimple *use_stmt;
|
||||
bool nested_in_vect_loop = false;
|
||||
auto_vec<gimple *> new_phis;
|
||||
auto_vec<stmt_vec_info> inner_phis;
|
||||
int j, i;
|
||||
auto_vec<tree> scalar_results;
|
||||
unsigned int group_size = 1, k, ratio;
|
||||
unsigned int group_size = 1, k;
|
||||
auto_vec<gimple *> phis;
|
||||
bool slp_reduc = false;
|
||||
bool direct_slp_reduc;
|
||||
tree new_phi_result;
|
||||
stmt_vec_info inner_phi = NULL;
|
||||
tree induction_index = NULL_TREE;
|
||||
|
||||
if (slp_node)
|
||||
|
@ -4327,28 +4326,15 @@ vect_create_epilog_for_reduction (stmt_vec_info stmt_info,
|
|||
nested_in_vect_loop = true;
|
||||
gcc_assert (!slp_node);
|
||||
}
|
||||
gcc_assert (!nested_in_vect_loop || double_reduc);
|
||||
|
||||
vectype = STMT_VINFO_VECTYPE (stmt_info);
|
||||
gcc_assert (vectype);
|
||||
mode = TYPE_MODE (vectype);
|
||||
|
||||
/* 1. Create the reduction def-use cycle:
|
||||
Set the backedge argument of REDUCTION_PHIS, i.e., transform
|
||||
|
||||
loop:
|
||||
vec_def = phi <vec_init, null> # REDUCTION_PHI
|
||||
VECT_DEF = vector_stmt # vectorized form of STMT
|
||||
...
|
||||
|
||||
into:
|
||||
|
||||
loop:
|
||||
vec_def = phi <vec_init, VECT_DEF> # REDUCTION_PHI
|
||||
VECT_DEF = vector_stmt # vectorized form of STMT
|
||||
...
|
||||
|
||||
(in case of SLP, do it for all the phis). */
|
||||
|
||||
tree initial_def = NULL;
|
||||
tree induc_val = NULL_TREE;
|
||||
tree adjustment_def = NULL;
|
||||
if (slp_node)
|
||||
;
|
||||
else
|
||||
|
@ -4357,9 +4343,8 @@ vect_create_epilog_for_reduction (stmt_vec_info stmt_info,
|
|||
of the reduction variable. */
|
||||
initial_def = PHI_ARG_DEF_FROM_EDGE (reduc_def_stmt,
|
||||
loop_preheader_edge (loop));
|
||||
/* Optimize: if initial_def is for REDUC_MAX smaller than the base
|
||||
and we can't use zero for induc_val, use initial_def. Similarly
|
||||
for REDUC_MIN and initial_def larger than the base. */
|
||||
/* Optimize: for induction condition reduction, if we can't use zero
|
||||
for induc_val, use initial_def. */
|
||||
if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info)
|
||||
== INTEGER_INDUC_COND_REDUCTION)
|
||||
induc_val = STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (stmt_info);
|
||||
|
@ -4371,49 +4356,25 @@ vect_create_epilog_for_reduction (stmt_vec_info stmt_info,
|
|||
adjustment_def = STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT (stmt_info);
|
||||
}
|
||||
|
||||
/* Set phi nodes latch arguments. */
|
||||
unsigned vec_num = 1;
|
||||
int ncopies = 0;
|
||||
unsigned vec_num;
|
||||
int ncopies;
|
||||
if (slp_node)
|
||||
vec_num = SLP_TREE_VEC_STMTS (slp_node_instance->reduc_phis).length ();
|
||||
for (unsigned i = 0; i < vec_num; i++)
|
||||
{
|
||||
if (slp_node)
|
||||
{
|
||||
def = gimple_get_lhs (SLP_TREE_VEC_STMTS (slp_node)[i]->stmt);
|
||||
phi_info = SLP_TREE_VEC_STMTS (slp_node_instance->reduc_phis)[i];
|
||||
}
|
||||
else
|
||||
{
|
||||
def = gimple_get_lhs (STMT_VINFO_VEC_STMT (stmt_info)->stmt);
|
||||
phi_info = STMT_VINFO_VEC_STMT (loop_vinfo->lookup_stmt (reduc_def_stmt));
|
||||
}
|
||||
for (ncopies = 0;; ncopies++)
|
||||
{
|
||||
/* Set the loop-latch arg for the reduction-phi. */
|
||||
if (ncopies != 0)
|
||||
{
|
||||
if (slp_node)
|
||||
break;
|
||||
phi_info = STMT_VINFO_RELATED_STMT (phi_info);
|
||||
if (!phi_info)
|
||||
break;
|
||||
def = vect_get_vec_def_for_stmt_copy (loop_vinfo, def);
|
||||
}
|
||||
|
||||
gphi *phi = as_a <gphi *> (phi_info->stmt);
|
||||
add_phi_arg (phi, def, loop_latch_edge (loop), UNKNOWN_LOCATION);
|
||||
|
||||
if (dump_enabled_p ())
|
||||
dump_printf_loc (MSG_NOTE, vect_location,
|
||||
"transform reduction: created def-use cycle: %G%G",
|
||||
phi, SSA_NAME_DEF_STMT (def));
|
||||
}
|
||||
vec_num = SLP_TREE_VEC_STMTS (slp_node_instance->reduc_phis).length ();
|
||||
ncopies = 1;
|
||||
}
|
||||
else
|
||||
{
|
||||
vec_num = 1;
|
||||
ncopies = 0;
|
||||
phi_info = STMT_VINFO_VEC_STMT (loop_vinfo->lookup_stmt (reduc_def_stmt));
|
||||
do
|
||||
{
|
||||
ncopies++;
|
||||
phi_info = STMT_VINFO_RELATED_STMT (phi_info);
|
||||
}
|
||||
while (phi_info);
|
||||
}
|
||||
|
||||
/* For vectorizing nested cycles the above is all we need to do. */
|
||||
if (nested_in_vect_loop && !double_reduc)
|
||||
return;
|
||||
|
||||
/* For cond reductions we want to create a new vector (INDEX_COND_EXPR)
|
||||
which is updated with the current index of the loop for every match of
|
||||
|
@ -4533,7 +4494,8 @@ vect_create_epilog_for_reduction (stmt_vec_info stmt_info,
|
|||
/* 2.1 Create new loop-exit-phis to preserve loop-closed form:
|
||||
v_out1 = phi <VECT_DEF>
|
||||
Store them in NEW_PHIS. */
|
||||
|
||||
if (double_reduc)
|
||||
loop = outer_loop;
|
||||
exit_bb = single_exit (loop)->dest;
|
||||
prev_phi_info = NULL;
|
||||
new_phis.create (slp_node ? vec_num : ncopies);
|
||||
|
@ -4542,7 +4504,7 @@ vect_create_epilog_for_reduction (stmt_vec_info stmt_info,
|
|||
if (slp_node)
|
||||
def = gimple_get_lhs (SLP_TREE_VEC_STMTS (slp_node)[i]->stmt);
|
||||
else
|
||||
def = gimple_get_lhs (STMT_VINFO_VEC_STMT (stmt_info)->stmt);
|
||||
def = gimple_get_lhs (STMT_VINFO_VEC_STMT (rdef_info)->stmt);
|
||||
for (j = 0; j < ncopies; j++)
|
||||
{
|
||||
tree new_def = copy_ssa_name (def);
|
||||
|
@ -4561,37 +4523,6 @@ vect_create_epilog_for_reduction (stmt_vec_info stmt_info,
|
|||
}
|
||||
}
|
||||
|
||||
/* The epilogue is created for the outer-loop, i.e., for the loop being
|
||||
vectorized. Create exit phis for the outer loop. */
|
||||
if (double_reduc)
|
||||
{
|
||||
loop = outer_loop;
|
||||
exit_bb = single_exit (loop)->dest;
|
||||
inner_phis.create (slp_node ? vec_num : ncopies);
|
||||
FOR_EACH_VEC_ELT (new_phis, i, phi)
|
||||
{
|
||||
stmt_vec_info phi_info = loop_vinfo->lookup_stmt (phi);
|
||||
tree new_result = copy_ssa_name (PHI_RESULT (phi));
|
||||
gphi *outer_phi = create_phi_node (new_result, exit_bb);
|
||||
SET_PHI_ARG_DEF (outer_phi, single_exit (loop)->dest_idx,
|
||||
PHI_RESULT (phi));
|
||||
prev_phi_info = loop_vinfo->add_stmt (outer_phi);
|
||||
inner_phis.quick_push (phi_info);
|
||||
new_phis[i] = outer_phi;
|
||||
while (STMT_VINFO_RELATED_STMT (phi_info))
|
||||
{
|
||||
phi_info = STMT_VINFO_RELATED_STMT (phi_info);
|
||||
new_result = copy_ssa_name (PHI_RESULT (phi_info->stmt));
|
||||
outer_phi = create_phi_node (new_result, exit_bb);
|
||||
SET_PHI_ARG_DEF (outer_phi, single_exit (loop)->dest_idx,
|
||||
PHI_RESULT (phi_info->stmt));
|
||||
stmt_vec_info outer_phi_info = loop_vinfo->add_stmt (outer_phi);
|
||||
STMT_VINFO_RELATED_STMT (prev_phi_info) = outer_phi_info;
|
||||
prev_phi_info = outer_phi_info;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
exit_gsi = gsi_after_labels (exit_bb);
|
||||
|
||||
/* 2.2 Get the relevant tree-code to use in the epilog for schemes 2,3
|
||||
|
@ -5381,23 +5312,10 @@ vect_create_epilog_for_reduction (stmt_vec_info stmt_info,
|
|||
correspond to the first vector stmt, etc.
|
||||
(RATIO is equal to (REDUC_GROUP_SIZE / number of new vector stmts)). */
|
||||
if (group_size > new_phis.length ())
|
||||
{
|
||||
ratio = group_size / new_phis.length ();
|
||||
gcc_assert (!(group_size % new_phis.length ()));
|
||||
}
|
||||
else
|
||||
ratio = 1;
|
||||
gcc_assert (!(group_size % new_phis.length ()));
|
||||
|
||||
stmt_vec_info epilog_stmt_info = NULL;
|
||||
for (k = 0; k < group_size; k++)
|
||||
{
|
||||
if (k % ratio == 0)
|
||||
{
|
||||
epilog_stmt_info = loop_vinfo->lookup_stmt (new_phis[k / ratio]);
|
||||
if (double_reduc)
|
||||
inner_phi = inner_phis[k / ratio];
|
||||
}
|
||||
|
||||
if (slp_reduc)
|
||||
{
|
||||
stmt_vec_info scalar_stmt_info = SLP_TREE_SCALAR_STMTS (slp_node)[k];
|
||||
|
@ -5408,84 +5326,6 @@ vect_create_epilog_for_reduction (stmt_vec_info stmt_info,
|
|||
scalar_dest = gimple_assign_lhs (scalar_stmt_info->stmt);
|
||||
}
|
||||
|
||||
if (outer_loop)
|
||||
{
|
||||
phis.create (3);
|
||||
/* Find the loop-closed-use at the loop exit of the original scalar
|
||||
result. (The reduction result is expected to have two immediate uses -
|
||||
one at the latch block, and one at the loop exit). */
|
||||
FOR_EACH_IMM_USE_FAST (use_p, imm_iter, scalar_dest)
|
||||
if (!flow_bb_inside_loop_p (loop, gimple_bb (USE_STMT (use_p)))
|
||||
&& !is_gimple_debug (USE_STMT (use_p)))
|
||||
phis.safe_push (USE_STMT (use_p));
|
||||
|
||||
/* While we expect to have found an exit_phi because of loop-closed-ssa
|
||||
form we can end up without one if the scalar cycle is dead. */
|
||||
|
||||
FOR_EACH_VEC_ELT (phis, i, exit_phi)
|
||||
{
|
||||
stmt_vec_info exit_phi_vinfo
|
||||
= loop_vinfo->lookup_stmt (exit_phi);
|
||||
gphi *vect_phi;
|
||||
|
||||
if (double_reduc)
|
||||
STMT_VINFO_VEC_STMT (exit_phi_vinfo) = inner_phi;
|
||||
else
|
||||
STMT_VINFO_VEC_STMT (exit_phi_vinfo) = epilog_stmt_info;
|
||||
if (!double_reduc
|
||||
|| STMT_VINFO_DEF_TYPE (exit_phi_vinfo)
|
||||
!= vect_double_reduction_def)
|
||||
continue;
|
||||
|
||||
/* Handle double reduction:
|
||||
|
||||
stmt1: s1 = phi <s0, s2> - double reduction phi (outer loop)
|
||||
stmt2: s3 = phi <s1, s4> - (regular) reduc phi (inner loop)
|
||||
stmt3: s4 = use (s3) - (regular) reduc stmt (inner loop)
|
||||
stmt4: s2 = phi <s4> - double reduction stmt (outer loop)
|
||||
|
||||
At that point the regular reduction (stmt2 and stmt3) is
|
||||
already vectorized, as well as the exit phi node, stmt4.
|
||||
Here we vectorize the phi node of double reduction, stmt1, and
|
||||
update all relevant statements. */
|
||||
|
||||
/* Go through all the uses of s2 to find double reduction phi
|
||||
node, i.e., stmt1 above. */
|
||||
orig_name = PHI_RESULT (exit_phi);
|
||||
FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, orig_name)
|
||||
{
|
||||
stmt_vec_info use_stmt_vinfo;
|
||||
basic_block bb = gimple_bb (use_stmt);
|
||||
|
||||
/* Check that USE_STMT is really double reduction phi
|
||||
node. */
|
||||
if (gimple_code (use_stmt) != GIMPLE_PHI
|
||||
|| gimple_phi_num_args (use_stmt) != 2
|
||||
|| bb->loop_father != outer_loop)
|
||||
continue;
|
||||
use_stmt_vinfo = loop_vinfo->lookup_stmt (use_stmt);
|
||||
if (!use_stmt_vinfo
|
||||
|| STMT_VINFO_DEF_TYPE (use_stmt_vinfo)
|
||||
!= vect_double_reduction_def)
|
||||
continue;
|
||||
|
||||
/* Set the outer loop vector phi args. The PHI node
|
||||
itself was created in vectorizable_reduction. */
|
||||
vect_phi = as_a <gphi *>
|
||||
(STMT_VINFO_VEC_STMT (use_stmt_vinfo)->stmt);
|
||||
|
||||
/* Update phi node argument with vs2. */
|
||||
add_phi_arg (vect_phi, PHI_RESULT (inner_phi->stmt),
|
||||
loop_latch_edge (outer_loop), UNKNOWN_LOCATION);
|
||||
if (dump_enabled_p ())
|
||||
dump_printf_loc (MSG_NOTE, vect_location,
|
||||
"created double reduction phi node: %G",
|
||||
vect_phi);
|
||||
}
|
||||
}
|
||||
phis.release ();
|
||||
}
|
||||
|
||||
if (nested_in_vect_loop)
|
||||
{
|
||||
if (double_reduc)
|
||||
|
@ -7153,10 +6993,6 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
|
|||
if (single_defuse_cycle && !slp_node)
|
||||
STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt_info;
|
||||
|
||||
/* Finalize the reduction-phi (set its arguments) and create the
|
||||
epilog reduction code. */
|
||||
vect_create_epilog_for_reduction (stmt_info, slp_node, slp_node_instance);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
@ -7172,10 +7008,8 @@ vectorizable_lc_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt,
|
|||
|| gimple_phi_num_args (stmt_info->stmt) != 1)
|
||||
return false;
|
||||
|
||||
/* To handle the nested_cycle_def for double-reductions we have to
|
||||
refactor epilogue generation more. */
|
||||
if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
|
||||
/* && STMT_VINFO_DEF_TYPE (stmt_info) != vect_double_reduction_def */)
|
||||
&& STMT_VINFO_DEF_TYPE (stmt_info) != vect_double_reduction_def)
|
||||
return false;
|
||||
|
||||
if (!vec_stmt) /* transformation not required. */
|
||||
|
@ -7833,8 +7667,8 @@ vectorizable_induction (stmt_vec_info stmt_info,
|
|||
bool
|
||||
vectorizable_live_operation (stmt_vec_info stmt_info,
|
||||
gimple_stmt_iterator *gsi ATTRIBUTE_UNUSED,
|
||||
slp_tree slp_node, int slp_index,
|
||||
stmt_vec_info *vec_stmt,
|
||||
slp_tree slp_node, slp_instance slp_node_instance,
|
||||
int slp_index, stmt_vec_info *vec_stmt,
|
||||
stmt_vector_for_cost *)
|
||||
{
|
||||
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
|
||||
|
@ -7851,8 +7685,34 @@ vectorizable_live_operation (stmt_vec_info stmt_info,
|
|||
|
||||
gcc_assert (STMT_VINFO_LIVE_P (stmt_info));
|
||||
|
||||
if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def)
|
||||
return false;
|
||||
/* The last stmt of a reduction is live and vectorized via
|
||||
vect_create_epilog_for_reduction. vectorizable_reduction assessed
|
||||
validity so just trigger the transform here. */
|
||||
if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def
|
||||
|| STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
|
||||
{
|
||||
if (!vec_stmt)
|
||||
return true;
|
||||
if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def)
|
||||
{
|
||||
if (STMT_VINFO_REDUC_TYPE (stmt_info) == FOLD_LEFT_REDUCTION)
|
||||
return true;
|
||||
if (slp_node)
|
||||
{
|
||||
/* For reduction chains the meta-info is attached to
|
||||
the group leader. */
|
||||
if (REDUC_GROUP_FIRST_ELEMENT (stmt_info))
|
||||
stmt_info = REDUC_GROUP_FIRST_ELEMENT (stmt_info);
|
||||
/* For SLP reductions we vectorize the epilogue for
|
||||
all involved stmts together. */
|
||||
else if (slp_index != 0)
|
||||
return true;
|
||||
}
|
||||
}
|
||||
vect_create_epilog_for_reduction (stmt_info, slp_node,
|
||||
slp_node_instance);
|
||||
return true;
|
||||
}
|
||||
|
||||
/* FORNOW. CHECKME. */
|
||||
if (nested_in_vect_loop_p (loop, stmt_info))
|
||||
|
|
|
@ -9809,19 +9809,17 @@ vectorizable_condition (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
|
|||
tree vec_cmp_type;
|
||||
bool masked = false;
|
||||
|
||||
if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
|
||||
return false;
|
||||
|
||||
if (for_reduction && STMT_SLP_TYPE (stmt_info))
|
||||
return false;
|
||||
|
||||
vect_reduction_type reduction_type
|
||||
= STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info);
|
||||
if (reduction_type == TREE_CODE_REDUCTION)
|
||||
if (!for_reduction)
|
||||
{
|
||||
if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
|
||||
return false;
|
||||
|
||||
if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
|
||||
&& !(STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle
|
||||
&& for_reduction))
|
||||
if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
|
||||
return false;
|
||||
|
||||
/* FORNOW: not yet supported. */
|
||||
|
@ -10472,7 +10470,8 @@ vectorizable_comparison (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
|
|||
|
||||
static bool
|
||||
can_vectorize_live_stmts (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
|
||||
slp_tree slp_node, stmt_vec_info *vec_stmt,
|
||||
slp_tree slp_node, slp_instance slp_node_instance,
|
||||
stmt_vec_info *vec_stmt,
|
||||
stmt_vector_for_cost *cost_vec)
|
||||
{
|
||||
if (slp_node)
|
||||
|
@ -10482,13 +10481,15 @@ can_vectorize_live_stmts (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
|
|||
FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (slp_node), i, slp_stmt_info)
|
||||
{
|
||||
if (STMT_VINFO_LIVE_P (slp_stmt_info)
|
||||
&& !vectorizable_live_operation (slp_stmt_info, gsi, slp_node, i,
|
||||
&& !vectorizable_live_operation (slp_stmt_info, gsi, slp_node,
|
||||
slp_node_instance, i,
|
||||
vec_stmt, cost_vec))
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else if (STMT_VINFO_LIVE_P (stmt_info)
|
||||
&& !vectorizable_live_operation (stmt_info, gsi, slp_node, -1,
|
||||
&& !vectorizable_live_operation (stmt_info, gsi, slp_node,
|
||||
slp_node_instance, -1,
|
||||
vec_stmt, cost_vec))
|
||||
return false;
|
||||
|
||||
|
@ -10704,7 +10705,9 @@ vect_analyze_stmt (stmt_vec_info stmt_info, bool *need_to_vectorize,
|
|||
need extra handling, except for vectorizable reductions. */
|
||||
if (!bb_vinfo
|
||||
&& STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type
|
||||
&& !can_vectorize_live_stmts (stmt_info, NULL, node, NULL, cost_vec))
|
||||
&& STMT_VINFO_TYPE (stmt_info) != lc_phi_info_type
|
||||
&& !can_vectorize_live_stmts (stmt_info, NULL, node, node_instance,
|
||||
NULL, cost_vec))
|
||||
return opt_result::failure_at (stmt_info->stmt,
|
||||
"not vectorized:"
|
||||
" live stmt not supported: %G",
|
||||
|
@ -10878,19 +10881,65 @@ vect_transform_stmt (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
|
|||
}
|
||||
}
|
||||
|
||||
/* Handle stmts whose DEF is used outside the loop-nest that is
|
||||
being vectorized. */
|
||||
if (STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type)
|
||||
{
|
||||
done = can_vectorize_live_stmts (stmt_info, gsi, slp_node, &vec_stmt,
|
||||
NULL);
|
||||
gcc_assert (done);
|
||||
}
|
||||
|
||||
if (vec_stmt)
|
||||
STMT_VINFO_VEC_STMT (stmt_info) = vec_stmt;
|
||||
|
||||
return is_store;
|
||||
if (STMT_VINFO_TYPE (stmt_info) == store_vec_info_type)
|
||||
return is_store;
|
||||
|
||||
/* If this stmt defines a value used on a backedge, update the
|
||||
vectorized PHIs. */
|
||||
stmt_vec_info orig_stmt_info = vect_orig_stmt (stmt_info);
|
||||
if (!slp_node && STMT_VINFO_REDUC_DEF (orig_stmt_info)
|
||||
&& STMT_VINFO_REDUC_TYPE (orig_stmt_info) != FOLD_LEFT_REDUCTION
|
||||
&& is_a <gphi *> (STMT_VINFO_REDUC_DEF (orig_stmt_info)->stmt))
|
||||
{
|
||||
gphi *phi = as_a <gphi *> (STMT_VINFO_REDUC_DEF (orig_stmt_info)->stmt);
|
||||
if (dominated_by_p (CDI_DOMINATORS,
|
||||
gimple_bb (orig_stmt_info->stmt), gimple_bb (phi)))
|
||||
{
|
||||
edge e = loop_latch_edge (gimple_bb (phi)->loop_father);
|
||||
stmt_vec_info phi_info
|
||||
= STMT_VINFO_VEC_STMT (STMT_VINFO_REDUC_DEF (orig_stmt_info));
|
||||
stmt_vec_info vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
|
||||
do
|
||||
{
|
||||
add_phi_arg (as_a <gphi *> (phi_info->stmt),
|
||||
gimple_get_lhs (vec_stmt->stmt), e,
|
||||
gimple_phi_arg_location (phi, e->dest_idx));
|
||||
phi_info = STMT_VINFO_RELATED_STMT (phi_info);
|
||||
vec_stmt = STMT_VINFO_RELATED_STMT (vec_stmt);
|
||||
}
|
||||
while (phi_info);
|
||||
gcc_assert (!vec_stmt);
|
||||
}
|
||||
}
|
||||
else if (slp_node && STMT_VINFO_REDUC_DEF (orig_stmt_info)
|
||||
/* Going back and forth via STMT_VINFO_REDUC_DEF gets us to the
|
||||
stmt with the reduction meta in case of reduction groups. */
|
||||
&& (STMT_VINFO_REDUC_TYPE
|
||||
(STMT_VINFO_REDUC_DEF (STMT_VINFO_REDUC_DEF (orig_stmt_info)))
|
||||
!= FOLD_LEFT_REDUCTION)
|
||||
&& slp_node != slp_node_instance->reduc_phis)
|
||||
{
|
||||
slp_tree phi_node = slp_node_instance->reduc_phis;
|
||||
gphi *phi = as_a <gphi *> (SLP_TREE_SCALAR_STMTS (phi_node)[0]->stmt);
|
||||
edge e = loop_latch_edge (gimple_bb (phi)->loop_father);
|
||||
gcc_assert (SLP_TREE_VEC_STMTS (phi_node).length ()
|
||||
== SLP_TREE_VEC_STMTS (slp_node).length ());
|
||||
for (unsigned i = 0; i < SLP_TREE_VEC_STMTS (phi_node).length (); ++i)
|
||||
add_phi_arg (as_a <gphi *> (SLP_TREE_VEC_STMTS (phi_node)[i]->stmt),
|
||||
gimple_get_lhs (SLP_TREE_VEC_STMTS (slp_node)[i]->stmt),
|
||||
e, gimple_phi_arg_location (phi, e->dest_idx));
|
||||
}
|
||||
|
||||
/* Handle stmts whose DEF is used outside the loop-nest that is
|
||||
being vectorized. */
|
||||
done = can_vectorize_live_stmts (stmt_info, gsi, slp_node,
|
||||
slp_node_instance, &vec_stmt, NULL);
|
||||
gcc_assert (done);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
|
|
|
@ -1649,7 +1649,8 @@ extern class loop *vect_transform_loop (loop_vec_info);
|
|||
extern opt_loop_vec_info vect_analyze_loop_form (class loop *,
|
||||
vec_info_shared *);
|
||||
extern bool vectorizable_live_operation (stmt_vec_info, gimple_stmt_iterator *,
|
||||
slp_tree, int, stmt_vec_info *,
|
||||
slp_tree, slp_instance, int,
|
||||
stmt_vec_info *,
|
||||
stmt_vector_for_cost *);
|
||||
extern bool vectorizable_reduction (stmt_vec_info, gimple_stmt_iterator *,
|
||||
stmt_vec_info *, slp_tree, slp_instance,
|
||||
|
|
Loading…
Reference in New Issue