diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 6638830bf80..c555f22f0e4 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,11 @@ +2016-10-14 Pat Haugen + + PR rtl-optimization/68212 + * cfgloopmanip.c (duplicate_loop_to_header_edge): Use preheader edge + frequency when computing scale factor for peeled copies. + * loop-unroll.c (unroll_loop_runtime_iterations): Fix freq/count + values for switch/peel blocks/edges. + 2016-10-14 Pedro Alves * coretypes.h (OVERRIDE, FINAL): Delete, moved to diff --git a/gcc/cfgloopmanip.c b/gcc/cfgloopmanip.c index 487b91df0e5..eb868393fac 100644 --- a/gcc/cfgloopmanip.c +++ b/gcc/cfgloopmanip.c @@ -1276,10 +1276,13 @@ duplicate_loop_to_header_edge (struct loop *loop, edge e, } else { + int preheader_freq = EDGE_FREQUENCY (e); scale_main = REG_BR_PROB_BASE; for (i = 0; i < ndupl; i++) scale_main = combine_probabilities (scale_main, scale_step[i]); - scale_act = REG_BR_PROB_BASE - prob_pass_thru; + if (preheader_freq > freq_in) + preheader_freq = freq_in; + scale_act = GCOV_COMPUTE_SCALE (preheader_freq, freq_in); } for (i = 0; i < ndupl; i++) gcc_assert (scale_step[i] >= 0 && scale_step[i] <= REG_BR_PROB_BASE); diff --git a/gcc/loop-unroll.c b/gcc/loop-unroll.c index 494504d61e6..f412698f170 100644 --- a/gcc/loop-unroll.c +++ b/gcc/loop-unroll.c @@ -859,7 +859,8 @@ unroll_loop_runtime_iterations (struct loop *loop) rtx_insn *init_code, *branch_code; unsigned i, j, p; basic_block preheader, *body, swtch, ezc_swtch; - int may_exit_copy; + int may_exit_copy, iter_freq, new_freq; + gcov_type iter_count, new_count; unsigned n_peel; edge e; bool extra_zero_check, last_may_exit; @@ -953,6 +954,15 @@ unroll_loop_runtime_iterations (struct loop *loop) /* Record the place where switch will be built for preconditioning. */ swtch = split_edge (loop_preheader_edge (loop)); + /* Compute frequency/count increments for each switch block and initialize + innermost switch block. Switch blocks and peeled loop copies are built + from innermost outward. */ + iter_freq = new_freq = swtch->frequency / (max_unroll + 1); + iter_count = new_count = swtch->count / (max_unroll + 1); + swtch->frequency = new_freq; + swtch->count = new_count; + single_succ_edge (swtch)->count = new_count; + for (i = 0; i < n_peel; i++) { /* Peel the copy. */ @@ -970,6 +980,10 @@ unroll_loop_runtime_iterations (struct loop *loop) p = REG_BR_PROB_BASE / (i + 2); preheader = split_edge (loop_preheader_edge (loop)); + /* Add in frequency/count of edge from switch block. */ + preheader->frequency += iter_freq; + preheader->count += iter_count; + single_succ_edge (preheader)->count = preheader->count; branch_code = compare_and_jump_seq (copy_rtx (niter), GEN_INT (j), EQ, block_label (preheader), p, NULL); @@ -981,9 +995,14 @@ unroll_loop_runtime_iterations (struct loop *loop) swtch = split_edge_and_insert (single_pred_edge (swtch), branch_code); set_immediate_dominator (CDI_DOMINATORS, preheader, swtch); single_succ_edge (swtch)->probability = REG_BR_PROB_BASE - p; + single_succ_edge (swtch)->count = new_count; + new_freq += iter_freq; + new_count += iter_count; + swtch->frequency = new_freq; + swtch->count = new_count; e = make_edge (swtch, preheader, single_succ_edge (swtch)->flags & EDGE_IRREDUCIBLE_LOOP); - e->count = RDIV (preheader->count * REG_BR_PROB_BASE, p); + e->count = iter_count; e->probability = p; } @@ -993,6 +1012,14 @@ unroll_loop_runtime_iterations (struct loop *loop) p = REG_BR_PROB_BASE / (max_unroll + 1); swtch = ezc_swtch; preheader = split_edge (loop_preheader_edge (loop)); + /* Recompute frequency/count adjustments since initial peel copy may + have exited and reduced those values that were computed above. */ + iter_freq = swtch->frequency / (max_unroll + 1); + iter_count = swtch->count / (max_unroll + 1); + /* Add in frequency/count of edge from switch block. */ + preheader->frequency += iter_freq; + preheader->count += iter_count; + single_succ_edge (preheader)->count = preheader->count; branch_code = compare_and_jump_seq (copy_rtx (niter), const0_rtx, EQ, block_label (preheader), p, NULL); @@ -1001,9 +1028,10 @@ unroll_loop_runtime_iterations (struct loop *loop) swtch = split_edge_and_insert (single_succ_edge (swtch), branch_code); set_immediate_dominator (CDI_DOMINATORS, preheader, swtch); single_succ_edge (swtch)->probability = REG_BR_PROB_BASE - p; + single_succ_edge (swtch)->count -= iter_count; e = make_edge (swtch, preheader, single_succ_edge (swtch)->flags & EDGE_IRREDUCIBLE_LOOP); - e->count = RDIV (preheader->count * REG_BR_PROB_BASE, p); + e->count = iter_count; e->probability = p; }