Makefile.in (tree-ssa-copy.o): Add $(CFGLOOP_H) dependency.
2009-04-04 Richard Guenther <rguenther@suse.de> * Makefile.in (tree-ssa-copy.o): Add $(CFGLOOP_H) dependency. * tree-ssa-copy.c (init_copy_prop): Do not propagate through single-argument PHIs if we are in loop-closed SSA form. * tree-vect-loop-manip.c (slpeel_add_loop_guard): Pass extra guards for the pre-condition. (slpeel_tree_peel_loop_to_edge): Likewise. (vect_build_loop_niters): Take an optional sequence to append stmts. (vect_generate_tmps_on_preheader): Likewise. (vect_do_peeling_for_loop_bound): Take extra guards for the pre-condition. (vect_do_peeling_for_alignment): Adjust. Unconditionally apply the cost model check. (vect_loop_versioning): Take stmt and stmt list to put pre-condition guards if we are going to peel. Do not apply versioning in that case. * tree-vectorizer.h (vect_loop_versioning): Adjust declaration. (vect_do_peeling_for_loop_bound): Likewise. * tree-vect-loop.c (vect_transform_loop): If we are peeling for loop bound only record extra pre-conditions, do not apply loop versioning. From-SVN: r145551
This commit is contained in:
Richard Guenther
Richard Biener
parent
a4d674fc53
commit
862900112e
@ -1,3 +1,26 @@
|
||||
2009-04-04 Richard Guenther <rguenther@suse.de>
|
||||
|
||||
* Makefile.in (tree-ssa-copy.o): Add $(CFGLOOP_H) dependency.
|
||||
* tree-ssa-copy.c (init_copy_prop): Do not propagate through
|
||||
single-argument PHIs if we are in loop-closed SSA form.
|
||||
* tree-vect-loop-manip.c (slpeel_add_loop_guard): Pass extra guards
|
||||
for the pre-condition.
|
||||
(slpeel_tree_peel_loop_to_edge): Likewise.
|
||||
(vect_build_loop_niters): Take an optional sequence to append stmts.
|
||||
(vect_generate_tmps_on_preheader): Likewise.
|
||||
(vect_do_peeling_for_loop_bound): Take extra guards for the
|
||||
pre-condition.
|
||||
(vect_do_peeling_for_alignment): Adjust. Unconditionally apply
|
||||
the cost model check.
|
||||
(vect_loop_versioning): Take stmt and stmt list to put pre-condition
|
||||
guards if we are going to peel. Do not apply versioning in that
|
||||
case.
|
||||
* tree-vectorizer.h (vect_loop_versioning): Adjust declaration.
|
||||
(vect_do_peeling_for_loop_bound): Likewise.
|
||||
* tree-vect-loop.c (vect_transform_loop): If we are peeling for
|
||||
loop bound only record extra pre-conditions, do not apply loop
|
||||
versioning.
|
||||
|
||||
2009-04-04 Richard Guenther <rguenther@suse.de>
|
||||
|
||||
* tree-ssa-operands.c (pop_stmt_changes): Remove automatic
|
||||
|
||||
@ -2124,7 +2124,8 @@ tree-nrv.o : tree-nrv.c $(CONFIG_H) $(SYSTEM_H) coretypes.h \
|
||||
tree-ssa-copy.o : tree-ssa-copy.c $(TREE_FLOW_H) $(CONFIG_H) $(SYSTEM_H) \
|
||||
$(RTL_H) $(TREE_H) $(TM_P_H) $(EXPR_H) $(GGC_H) output.h $(DIAGNOSTIC_H) \
|
||||
$(FUNCTION_H) $(TIMEVAR_H) $(TM_H) coretypes.h $(TREE_DUMP_H) \
|
||||
$(BASIC_BLOCK_H) tree-pass.h langhooks.h tree-ssa-propagate.h $(FLAGS_H)
|
||||
$(BASIC_BLOCK_H) tree-pass.h langhooks.h tree-ssa-propagate.h $(FLAGS_H) \
|
||||
$(CFGLOOP_H)
|
||||
tree-ssa-propagate.o : tree-ssa-propagate.c $(TREE_FLOW_H) $(CONFIG_H) \
|
||||
$(SYSTEM_H) $(RTL_H) $(TREE_H) $(TM_P_H) $(EXPR_H) $(GGC_H) output.h \
|
||||
$(DIAGNOSTIC_H) $(FUNCTION_H) $(TIMEVAR_H) $(TM_H) coretypes.h \
|
||||
|
||||
@ -37,6 +37,7 @@ along with GCC; see the file COPYING3. If not see
|
||||
#include "tree-pass.h"
|
||||
#include "tree-ssa-propagate.h"
|
||||
#include "langhooks.h"
|
||||
#include "cfgloop.h"
|
||||
|
||||
/* This file implements the copy propagation pass and provides a
|
||||
handful of interfaces for performing const/copy propagation and
|
||||
@ -885,7 +886,13 @@ init_copy_prop (void)
|
||||
tree def;
|
||||
|
||||
def = gimple_phi_result (phi);
|
||||
if (!is_gimple_reg (def))
|
||||
if (!is_gimple_reg (def)
|
||||
/* In loop-closed SSA form do not copy-propagate through
|
||||
PHI nodes. Technically this is only needed for loop
|
||||
exit PHIs, but this is difficult to query. */
|
||||
|| (current_loops
|
||||
&& gimple_phi_num_args (phi) == 1
|
||||
&& loops_state_satisfies_p (LOOP_CLOSED_SSA)))
|
||||
prop_set_simulate_again (phi, false);
|
||||
else
|
||||
prop_set_simulate_again (phi, true);
|
||||
|
||||
@ -792,11 +792,12 @@ slpeel_tree_duplicate_loop_to_edge_cfg (struct loop *loop, edge e)
|
||||
/* Given the condition statement COND, put it as the last statement
|
||||
of GUARD_BB; EXIT_BB is the basic block to skip the loop;
|
||||
Assumes that this is the single exit of the guarded loop.
|
||||
Returns the skip edge. */
|
||||
Returns the skip edge, inserts new stmts on the COND_EXPR_STMT_LIST. */
|
||||
|
||||
static edge
|
||||
slpeel_add_loop_guard (basic_block guard_bb, tree cond, basic_block exit_bb,
|
||||
basic_block dom_bb)
|
||||
slpeel_add_loop_guard (basic_block guard_bb, tree cond,
|
||||
gimple_seq cond_expr_stmt_list,
|
||||
basic_block exit_bb, basic_block dom_bb)
|
||||
{
|
||||
gimple_stmt_iterator gsi;
|
||||
edge new_e, enter_e;
|
||||
@ -809,11 +810,13 @@ slpeel_add_loop_guard (basic_block guard_bb, tree cond, basic_block exit_bb,
|
||||
gsi = gsi_last_bb (guard_bb);
|
||||
|
||||
cond = force_gimple_operand (cond, &gimplify_stmt_list, true, NULL_TREE);
|
||||
if (gimplify_stmt_list)
|
||||
gimple_seq_add_seq (&cond_expr_stmt_list, gimplify_stmt_list);
|
||||
cond_stmt = gimple_build_cond (NE_EXPR,
|
||||
cond, build_int_cst (TREE_TYPE (cond), 0),
|
||||
NULL_TREE, NULL_TREE);
|
||||
if (gimplify_stmt_list)
|
||||
gsi_insert_seq_after (&gsi, gimplify_stmt_list, GSI_NEW_STMT);
|
||||
if (cond_expr_stmt_list)
|
||||
gsi_insert_seq_after (&gsi, cond_expr_stmt_list, GSI_NEW_STMT);
|
||||
|
||||
gsi = gsi_last_bb (guard_bb);
|
||||
gsi_insert_after (&gsi, cond_stmt, GSI_NEW_STMT);
|
||||
@ -1011,6 +1014,10 @@ set_prologue_iterations (basic_block bb_before_first_loop,
|
||||
The second guard is:
|
||||
if (FIRST_NITERS == NITERS) then skip the second loop.
|
||||
|
||||
If the optional COND_EXPR and COND_EXPR_STMT_LIST arguments are given
|
||||
then the generated condition is combined with COND_EXPR and the
|
||||
statements in COND_EXPR_STMT_LIST are emitted together with it.
|
||||
|
||||
FORNOW only simple loops are supported (see slpeel_can_duplicate_loop_p).
|
||||
FORNOW the resulting code will not be in loop-closed-ssa form.
|
||||
*/
|
||||
@ -1019,7 +1026,8 @@ static struct loop*
|
||||
slpeel_tree_peel_loop_to_edge (struct loop *loop,
|
||||
edge e, tree first_niters,
|
||||
tree niters, bool update_first_loop_count,
|
||||
unsigned int th, bool check_profitability)
|
||||
unsigned int th, bool check_profitability,
|
||||
tree cond_expr, gimple_seq cond_expr_stmt_list)
|
||||
{
|
||||
struct loop *new_loop = NULL, *first_loop, *second_loop;
|
||||
edge skip_e;
|
||||
@ -1149,7 +1157,8 @@ slpeel_tree_peel_loop_to_edge (struct loop *loop,
|
||||
profitable than the vector one. This occurs when
|
||||
this function is invoked for epilogue generation
|
||||
and the cost model check needs to be done at run
|
||||
time.
|
||||
time. This check is combined with any pre-existing
|
||||
check in COND_EXPR to avoid versioning.
|
||||
|
||||
Resulting CFG after prologue peeling would be:
|
||||
|
||||
@ -1193,6 +1202,14 @@ slpeel_tree_peel_loop_to_edge (struct loop *loop,
|
||||
pre_condition = fold_build2 (TRUTH_OR_EXPR, boolean_type_node,
|
||||
cost_pre_condition, pre_condition);
|
||||
}
|
||||
if (cond_expr)
|
||||
{
|
||||
pre_condition =
|
||||
fold_build2 (TRUTH_OR_EXPR, boolean_type_node,
|
||||
pre_condition,
|
||||
fold_build1 (TRUTH_NOT_EXPR, boolean_type_node,
|
||||
cond_expr));
|
||||
}
|
||||
}
|
||||
|
||||
/* Prologue peeling. */
|
||||
@ -1208,6 +1225,7 @@ slpeel_tree_peel_loop_to_edge (struct loop *loop,
|
||||
}
|
||||
|
||||
skip_e = slpeel_add_loop_guard (bb_before_first_loop, pre_condition,
|
||||
cond_expr_stmt_list,
|
||||
bb_before_second_loop, bb_before_first_loop);
|
||||
slpeel_update_phi_nodes_for_guard1 (skip_e, first_loop,
|
||||
first_loop == new_loop,
|
||||
@ -1245,7 +1263,7 @@ slpeel_tree_peel_loop_to_edge (struct loop *loop,
|
||||
|
||||
pre_condition =
|
||||
fold_build2 (EQ_EXPR, boolean_type_node, first_niters, niters);
|
||||
skip_e = slpeel_add_loop_guard (bb_between_loops, pre_condition,
|
||||
skip_e = slpeel_add_loop_guard (bb_between_loops, pre_condition, NULL,
|
||||
bb_after_second_loop, bb_before_first_loop);
|
||||
slpeel_update_phi_nodes_for_guard2 (skip_e, second_loop,
|
||||
second_loop == new_loop, &new_exit_bb);
|
||||
@ -1303,10 +1321,11 @@ find_loop_location (struct loop *loop)
|
||||
|
||||
|
||||
/* This function builds ni_name = number of iterations loop executes
|
||||
on the loop preheader. */
|
||||
on the loop preheader. If SEQ is given the stmt is instead emitted
|
||||
there. */
|
||||
|
||||
static tree
|
||||
vect_build_loop_niters (loop_vec_info loop_vinfo)
|
||||
vect_build_loop_niters (loop_vec_info loop_vinfo, gimple_seq seq)
|
||||
{
|
||||
tree ni_name, var;
|
||||
gimple_seq stmts = NULL;
|
||||
@ -1321,8 +1340,13 @@ vect_build_loop_niters (loop_vec_info loop_vinfo)
|
||||
pe = loop_preheader_edge (loop);
|
||||
if (stmts)
|
||||
{
|
||||
basic_block new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
|
||||
gcc_assert (!new_bb);
|
||||
if (seq)
|
||||
gimple_seq_add_seq (&seq, stmts);
|
||||
else
|
||||
{
|
||||
basic_block new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
|
||||
gcc_assert (!new_bb);
|
||||
}
|
||||
}
|
||||
|
||||
return ni_name;
|
||||
@ -1335,13 +1359,15 @@ vect_build_loop_niters (loop_vec_info loop_vinfo)
|
||||
ratio = ni_name / vf
|
||||
ratio_mult_vf_name = ratio * vf
|
||||
|
||||
and places them at the loop preheader edge. */
|
||||
and places them at the loop preheader edge or in COND_EXPR_STMT_LIST
|
||||
if that is non-NULL. */
|
||||
|
||||
static void
|
||||
vect_generate_tmps_on_preheader (loop_vec_info loop_vinfo,
|
||||
tree *ni_name_ptr,
|
||||
tree *ratio_mult_vf_name_ptr,
|
||||
tree *ratio_name_ptr)
|
||||
tree *ratio_name_ptr,
|
||||
gimple_seq cond_expr_stmt_list)
|
||||
{
|
||||
|
||||
edge pe;
|
||||
@ -1361,7 +1387,7 @@ vect_generate_tmps_on_preheader (loop_vec_info loop_vinfo,
|
||||
/* Generate temporary variable that contains
|
||||
number of iterations loop executes. */
|
||||
|
||||
ni_name = vect_build_loop_niters (loop_vinfo);
|
||||
ni_name = vect_build_loop_niters (loop_vinfo, cond_expr_stmt_list);
|
||||
log_vf = build_int_cst (TREE_TYPE (ni), exact_log2 (vf));
|
||||
|
||||
/* Create: ratio = ni >> log2(vf) */
|
||||
@ -1374,9 +1400,14 @@ vect_generate_tmps_on_preheader (loop_vec_info loop_vinfo,
|
||||
|
||||
stmts = NULL;
|
||||
ratio_name = force_gimple_operand (ratio_name, &stmts, true, var);
|
||||
pe = loop_preheader_edge (loop);
|
||||
new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
|
||||
gcc_assert (!new_bb);
|
||||
if (cond_expr_stmt_list)
|
||||
gimple_seq_add_seq (&cond_expr_stmt_list, stmts);
|
||||
else
|
||||
{
|
||||
pe = loop_preheader_edge (loop);
|
||||
new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
|
||||
gcc_assert (!new_bb);
|
||||
}
|
||||
}
|
||||
|
||||
/* Create: ratio_mult_vf = ratio << log2 (vf). */
|
||||
@ -1391,9 +1422,14 @@ vect_generate_tmps_on_preheader (loop_vec_info loop_vinfo,
|
||||
stmts = NULL;
|
||||
ratio_mult_vf_name = force_gimple_operand (ratio_mult_vf_name, &stmts,
|
||||
true, var);
|
||||
pe = loop_preheader_edge (loop);
|
||||
new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
|
||||
gcc_assert (!new_bb);
|
||||
if (cond_expr_stmt_list)
|
||||
gimple_seq_add_seq (&cond_expr_stmt_list, stmts);
|
||||
else
|
||||
{
|
||||
pe = loop_preheader_edge (loop);
|
||||
new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
|
||||
gcc_assert (!new_bb);
|
||||
}
|
||||
}
|
||||
|
||||
*ni_name_ptr = ni_name;
|
||||
@ -1664,10 +1700,14 @@ conservative_cost_threshold (loop_vec_info loop_vinfo,
|
||||
NITERS % VECTORIZATION_FACTOR times.
|
||||
|
||||
The original loop will later be made to iterate
|
||||
NITERS / VECTORIZATION_FACTOR times (this value is placed into RATIO). */
|
||||
NITERS / VECTORIZATION_FACTOR times (this value is placed into RATIO).
|
||||
|
||||
COND_EXPR and COND_EXPR_STMT_LIST are combined with a new generated
|
||||
test. */
|
||||
|
||||
void
|
||||
vect_do_peeling_for_loop_bound (loop_vec_info loop_vinfo, tree *ratio)
|
||||
vect_do_peeling_for_loop_bound (loop_vec_info loop_vinfo, tree *ratio,
|
||||
tree cond_expr, gimple_seq cond_expr_stmt_list)
|
||||
{
|
||||
tree ni_name, ratio_mult_vf_name;
|
||||
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
||||
@ -1690,7 +1730,8 @@ vect_do_peeling_for_loop_bound (loop_vec_info loop_vinfo, tree *ratio)
|
||||
ratio = ni_name / vf
|
||||
ratio_mult_vf_name = ratio * vf */
|
||||
vect_generate_tmps_on_preheader (loop_vinfo, &ni_name,
|
||||
&ratio_mult_vf_name, ratio);
|
||||
&ratio_mult_vf_name, ratio,
|
||||
cond_expr_stmt_list);
|
||||
|
||||
loop_num = loop->num;
|
||||
|
||||
@ -1698,7 +1739,8 @@ vect_do_peeling_for_loop_bound (loop_vec_info loop_vinfo, tree *ratio)
|
||||
peeling for alignment. */
|
||||
if (!VEC_length (gimple, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo))
|
||||
&& !VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo))
|
||||
&& !LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo))
|
||||
&& !LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo)
|
||||
&& !cond_expr)
|
||||
{
|
||||
check_profitability = true;
|
||||
|
||||
@ -1711,7 +1753,8 @@ vect_do_peeling_for_loop_bound (loop_vec_info loop_vinfo, tree *ratio)
|
||||
|
||||
new_loop = slpeel_tree_peel_loop_to_edge (loop, single_exit (loop),
|
||||
ratio_mult_vf_name, ni_name, false,
|
||||
th, check_profitability);
|
||||
th, check_profitability,
|
||||
cond_expr, cond_expr_stmt_list);
|
||||
gcc_assert (new_loop);
|
||||
gcc_assert (loop_num == loop->num);
|
||||
#ifdef ENABLE_CHECKING
|
||||
@ -1926,7 +1969,6 @@ vect_do_peeling_for_alignment (loop_vec_info loop_vinfo)
|
||||
tree niters_of_prolog_loop, ni_name;
|
||||
tree n_iters;
|
||||
struct loop *new_loop;
|
||||
bool check_profitability = false;
|
||||
unsigned int th = 0;
|
||||
int min_profitable_iters;
|
||||
|
||||
@ -1935,28 +1977,20 @@ vect_do_peeling_for_alignment (loop_vec_info loop_vinfo)
|
||||
|
||||
initialize_original_copy_tables ();
|
||||
|
||||
ni_name = vect_build_loop_niters (loop_vinfo);
|
||||
ni_name = vect_build_loop_niters (loop_vinfo, NULL);
|
||||
niters_of_prolog_loop = vect_gen_niters_for_prolog_loop (loop_vinfo, ni_name);
|
||||
|
||||
|
||||
/* If cost model check not done during versioning. */
|
||||
if (!VEC_length (gimple, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo))
|
||||
&& !VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo)))
|
||||
{
|
||||
check_profitability = true;
|
||||
|
||||
/* Get profitability threshold for vectorized loop. */
|
||||
min_profitable_iters = LOOP_VINFO_COST_MODEL_MIN_ITERS (loop_vinfo);
|
||||
|
||||
th = conservative_cost_threshold (loop_vinfo,
|
||||
min_profitable_iters);
|
||||
}
|
||||
/* Get profitability threshold for vectorized loop. */
|
||||
min_profitable_iters = LOOP_VINFO_COST_MODEL_MIN_ITERS (loop_vinfo);
|
||||
th = conservative_cost_threshold (loop_vinfo,
|
||||
min_profitable_iters);
|
||||
|
||||
/* Peel the prolog loop and iterate it niters_of_prolog_loop. */
|
||||
new_loop =
|
||||
slpeel_tree_peel_loop_to_edge (loop, loop_preheader_edge (loop),
|
||||
niters_of_prolog_loop, ni_name, true,
|
||||
th, check_profitability);
|
||||
th, true, NULL_TREE, NULL);
|
||||
|
||||
gcc_assert (new_loop);
|
||||
#ifdef ENABLE_CHECKING
|
||||
@ -2273,15 +2307,18 @@ vect_create_cond_for_alias_checks (loop_vec_info loop_vinfo,
|
||||
|
||||
The test generated to check which version of loop is executed
|
||||
is modified to also check for profitability as indicated by the
|
||||
cost model initially. */
|
||||
cost model initially.
|
||||
|
||||
The versioning precondition(s) are placed in *COND_EXPR and
|
||||
*COND_EXPR_STMT_LIST. If DO_VERSIONING is true versioning is
|
||||
also performed, otherwise only the conditions are generated. */
|
||||
|
||||
void
|
||||
vect_loop_versioning (loop_vec_info loop_vinfo)
|
||||
vect_loop_versioning (loop_vec_info loop_vinfo, bool do_versioning,
|
||||
tree *cond_expr, gimple_seq *cond_expr_stmt_list)
|
||||
{
|
||||
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
||||
struct loop *nloop;
|
||||
tree cond_expr = NULL_TREE;
|
||||
gimple_seq cond_expr_stmt_list = NULL;
|
||||
basic_block condition_bb;
|
||||
gimple_stmt_iterator gsi, cond_exp_gsi;
|
||||
basic_block merge_bb;
|
||||
@ -2301,29 +2338,34 @@ vect_loop_versioning (loop_vec_info loop_vinfo)
|
||||
th = conservative_cost_threshold (loop_vinfo,
|
||||
min_profitable_iters);
|
||||
|
||||
cond_expr =
|
||||
*cond_expr =
|
||||
fold_build2 (GT_EXPR, boolean_type_node, scalar_loop_iters,
|
||||
build_int_cst (TREE_TYPE (scalar_loop_iters), th));
|
||||
|
||||
cond_expr = force_gimple_operand (cond_expr, &cond_expr_stmt_list,
|
||||
false, NULL_TREE);
|
||||
*cond_expr = force_gimple_operand (*cond_expr, cond_expr_stmt_list,
|
||||
false, NULL_TREE);
|
||||
|
||||
if (VEC_length (gimple, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo)))
|
||||
vect_create_cond_for_align_checks (loop_vinfo, &cond_expr,
|
||||
&cond_expr_stmt_list);
|
||||
vect_create_cond_for_align_checks (loop_vinfo, cond_expr,
|
||||
cond_expr_stmt_list);
|
||||
|
||||
if (VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo)))
|
||||
vect_create_cond_for_alias_checks (loop_vinfo, &cond_expr,
|
||||
&cond_expr_stmt_list);
|
||||
vect_create_cond_for_alias_checks (loop_vinfo, cond_expr,
|
||||
cond_expr_stmt_list);
|
||||
|
||||
cond_expr =
|
||||
fold_build2 (NE_EXPR, boolean_type_node, cond_expr, integer_zero_node);
|
||||
cond_expr =
|
||||
force_gimple_operand (cond_expr, &gimplify_stmt_list, true, NULL_TREE);
|
||||
gimple_seq_add_seq (&cond_expr_stmt_list, gimplify_stmt_list);
|
||||
*cond_expr =
|
||||
fold_build2 (NE_EXPR, boolean_type_node, *cond_expr, integer_zero_node);
|
||||
*cond_expr =
|
||||
force_gimple_operand (*cond_expr, &gimplify_stmt_list, true, NULL_TREE);
|
||||
gimple_seq_add_seq (cond_expr_stmt_list, gimplify_stmt_list);
|
||||
|
||||
/* If we only needed the extra conditions and a new loop copy
|
||||
bail out here. */
|
||||
if (!do_versioning)
|
||||
return;
|
||||
|
||||
initialize_original_copy_tables ();
|
||||
nloop = loop_version (loop, cond_expr, &condition_bb,
|
||||
nloop = loop_version (loop, *cond_expr, &condition_bb,
|
||||
prob, prob, REG_BR_PROB_BASE - prob, true);
|
||||
free_original_copy_tables();
|
||||
|
||||
@ -2354,10 +2396,13 @@ vect_loop_versioning (loop_vec_info loop_vinfo)
|
||||
/* End loop-exit-fixes after versioning. */
|
||||
|
||||
update_ssa (TODO_update_ssa);
|
||||
if (cond_expr_stmt_list)
|
||||
if (*cond_expr_stmt_list)
|
||||
{
|
||||
cond_exp_gsi = gsi_last_bb (condition_bb);
|
||||
gsi_insert_seq_before (&cond_exp_gsi, cond_expr_stmt_list, GSI_SAME_STMT);
|
||||
gsi_insert_seq_before (&cond_exp_gsi, *cond_expr_stmt_list,
|
||||
GSI_SAME_STMT);
|
||||
*cond_expr_stmt_list = NULL;
|
||||
}
|
||||
*cond_expr = NULL_TREE;
|
||||
}
|
||||
|
||||
|
||||
@ -3388,23 +3388,33 @@ vect_transform_loop (loop_vec_info loop_vinfo)
|
||||
bool strided_store;
|
||||
bool slp_scheduled = false;
|
||||
unsigned int nunits;
|
||||
tree cond_expr = NULL_TREE;
|
||||
gimple_seq cond_expr_stmt_list = NULL;
|
||||
bool do_peeling_for_loop_bound;
|
||||
|
||||
if (vect_print_dump_info (REPORT_DETAILS))
|
||||
fprintf (vect_dump, "=== vec_transform_loop ===");
|
||||
|
||||
if (VEC_length (gimple, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo))
|
||||
|| VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo)))
|
||||
vect_loop_versioning (loop_vinfo);
|
||||
|
||||
/* CHECKME: we wouldn't need this if we called update_ssa once
|
||||
for all loops. */
|
||||
bitmap_zero (vect_memsyms_to_rename);
|
||||
|
||||
/* Peel the loop if there are data refs with unknown alignment.
|
||||
Only one data ref with unknown store is allowed. */
|
||||
|
||||
if (LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo))
|
||||
vect_do_peeling_for_alignment (loop_vinfo);
|
||||
|
||||
do_peeling_for_loop_bound
|
||||
= (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
|
||||
|| (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
|
||||
&& LOOP_VINFO_INT_NITERS (loop_vinfo) % vectorization_factor != 0));
|
||||
|
||||
if (VEC_length (gimple, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo))
|
||||
|| VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo)))
|
||||
vect_loop_versioning (loop_vinfo,
|
||||
!do_peeling_for_loop_bound,
|
||||
&cond_expr, &cond_expr_stmt_list);
|
||||
|
||||
/* CHECKME: we wouldn't need this if we called update_ssa once
|
||||
for all loops. */
|
||||
bitmap_zero (vect_memsyms_to_rename);
|
||||
|
||||
/* If the loop has a symbolic number of iterations 'n' (i.e. it's not a
|
||||
compile time constant), or it is a constant that doesn't divide by the
|
||||
@ -3414,10 +3424,9 @@ vect_transform_loop (loop_vec_info loop_vinfo)
|
||||
will remain scalar and will compute the remaining (n%VF) iterations.
|
||||
(VF is the vectorization factor). */
|
||||
|
||||
if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
|
||||
|| (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
|
||||
&& LOOP_VINFO_INT_NITERS (loop_vinfo) % vectorization_factor != 0))
|
||||
vect_do_peeling_for_loop_bound (loop_vinfo, &ratio);
|
||||
if (do_peeling_for_loop_bound)
|
||||
vect_do_peeling_for_loop_bound (loop_vinfo, &ratio,
|
||||
cond_expr, cond_expr_stmt_list);
|
||||
else
|
||||
ratio = build_int_cst (TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo)),
|
||||
LOOP_VINFO_INT_NITERS (loop_vinfo) / vectorization_factor);
|
||||
|
||||
@ -706,8 +706,9 @@ extern bitmap vect_memsyms_to_rename;
|
||||
in tree-vect-loop-manip.c. */
|
||||
extern void slpeel_make_loop_iterate_ntimes (struct loop *, tree);
|
||||
extern bool slpeel_can_duplicate_loop_p (const struct loop *, const_edge);
|
||||
extern void vect_loop_versioning (loop_vec_info);
|
||||
extern void vect_do_peeling_for_loop_bound (loop_vec_info, tree *);
|
||||
extern void vect_loop_versioning (loop_vec_info, bool, tree *, gimple_seq *);
|
||||
extern void vect_do_peeling_for_loop_bound (loop_vec_info, tree *,
|
||||
tree, gimple_seq);
|
||||
extern void vect_do_peeling_for_alignment (loop_vec_info);
|
||||
extern LOC find_loop_location (struct loop *);
|
||||
extern bool vect_can_advance_ivs_p (loop_vec_info);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user