Fix profile count maintenance in vectorizer peeling.
This patch changes the code to save/restore profile counts for the epliog loop (when not using scalar loop in the epilog) instead of scaling them down and then back up, which may lead to problems if we scale down to 0. Tested on x86_64-pc-linux-gnu. gcc/ChangeLog: * tree-vect-loop-manip.cc (vect_do_peeling): Save/restore profile counts for the epilog loop.
This commit is contained in:
parent
10d1986aee
commit
5af22024f6
|
@ -2810,10 +2810,21 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1,
|
||||||
if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo))
|
if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo))
|
||||||
skip_epilog = false;
|
skip_epilog = false;
|
||||||
|
|
||||||
|
class loop *scalar_loop = LOOP_VINFO_SCALAR_LOOP (loop_vinfo);
|
||||||
|
auto_vec<profile_count> original_counts;
|
||||||
|
basic_block *original_bbs = NULL;
|
||||||
|
|
||||||
if (skip_vector)
|
if (skip_vector)
|
||||||
{
|
{
|
||||||
split_edge (loop_preheader_edge (loop));
|
split_edge (loop_preheader_edge (loop));
|
||||||
|
|
||||||
|
if (epilog_peeling && (vect_epilogues || scalar_loop == NULL))
|
||||||
|
{
|
||||||
|
original_bbs = get_loop_body (loop);
|
||||||
|
for (unsigned int i = 0; i < loop->num_nodes; i++)
|
||||||
|
original_counts.safe_push(original_bbs[i]->count);
|
||||||
|
}
|
||||||
|
|
||||||
/* Due to the order in which we peel prolog and epilog, we first
|
/* Due to the order in which we peel prolog and epilog, we first
|
||||||
propagate probability to the whole loop. The purpose is to
|
propagate probability to the whole loop. The purpose is to
|
||||||
avoid adjusting probabilities of both prolog and vector loops
|
avoid adjusting probabilities of both prolog and vector loops
|
||||||
|
@ -2828,7 +2839,6 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1,
|
||||||
}
|
}
|
||||||
|
|
||||||
dump_user_location_t loop_loc = find_loop_location (loop);
|
dump_user_location_t loop_loc = find_loop_location (loop);
|
||||||
class loop *scalar_loop = LOOP_VINFO_SCALAR_LOOP (loop_vinfo);
|
|
||||||
if (vect_epilogues)
|
if (vect_epilogues)
|
||||||
/* Make sure to set the epilogue's epilogue scalar loop, such that we can
|
/* Make sure to set the epilogue's epilogue scalar loop, such that we can
|
||||||
use the original scalar loop as remaining epilogue if necessary. */
|
use the original scalar loop as remaining epilogue if necessary. */
|
||||||
|
@ -2985,16 +2995,19 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1,
|
||||||
a merge point of control flow. */
|
a merge point of control flow. */
|
||||||
guard_to->count = guard_bb->count;
|
guard_to->count = guard_bb->count;
|
||||||
|
|
||||||
/* Scale probability of epilog loop back.
|
/* Restore the counts of the epilog loop if we didn't use the scalar loop. */
|
||||||
FIXME: We should avoid scaling down and back up. Profile may
|
if (vect_epilogues || scalar_loop == NULL)
|
||||||
get lost if we scale down to 0. */
|
{
|
||||||
|
gcc_assert(epilog->num_nodes == loop->num_nodes);
|
||||||
basic_block *bbs = get_loop_body (epilog);
|
basic_block *bbs = get_loop_body (epilog);
|
||||||
for (unsigned int i = 0; i < epilog->num_nodes; i++)
|
for (unsigned int i = 0; i < epilog->num_nodes; i++)
|
||||||
bbs[i]->count = bbs[i]->count.apply_scale
|
{
|
||||||
(bbs[i]->count,
|
gcc_assert(get_bb_original (bbs[i]) == original_bbs[i]);
|
||||||
bbs[i]->count.apply_probability
|
bbs[i]->count = original_counts[i];
|
||||||
(prob_vector));
|
}
|
||||||
free (bbs);
|
free (bbs);
|
||||||
|
free (original_bbs);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
basic_block bb_before_epilog = loop_preheader_edge (epilog)->src;
|
basic_block bb_before_epilog = loop_preheader_edge (epilog)->src;
|
||||||
|
|
Loading…
Reference in New Issue