diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 9796fa09ce9..bee2ebc42d7 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,25 @@ +2009-11-14 Jan Hubicka + + * cgraphbuild.c (compute_call_stmt_bb_frequency): Use proper ENTRY_BLOCK_PTR. + * cgraph.c (cgraph_clone_edge): Avoid freq_scale 0 to completely zero out all + callees. + * cgraphunit.c (verify_cgraph_node): Verify cgraph nodes for frequency and count match. + * ipa-inline.c (update_noncloned_frequencies): New function. + (cgraph_clone_inlined_nodes): Use it. + * tree-inline.c (copy_bb): Fix frequency scaling; output + diagnostic on frequency mismatches to dump file. + (initialize_cfun): Do not scale frequency; fix count scaling; + initialize entry and exit block frequencies; copy profile + info. + (copy_cfg_body): Use frequency_scale as argument; + fix count scaling. + (copy_body): Use frequency_scale as argument. + (expand_call_inline): Compute frequency scale and output diagnostic + to dump file. + (delete_unreachable_blocks_update_callgrah): Remove checking that + has to be done after edge redirection. + (tree_function_versioning): Update initialize_cfun and copy_body call. + 2009-11-14 Jan Hubicka * cgraph.c (cgraph_release_function_body): Update use of diff --git a/gcc/cgraph.c b/gcc/cgraph.c index 840cf297bb5..f2d76091799 100644 --- a/gcc/cgraph.c +++ b/gcc/cgraph.c @@ -1641,8 +1641,12 @@ cgraph_clone_edge (struct cgraph_edge *e, struct cgraph_node *n, { struct cgraph_edge *new_edge; gcov_type count = e->count * count_scale / REG_BR_PROB_BASE; - gcov_type freq = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE; + gcov_type freq; + /* We do not want to ignore loop nest after frequency drops to 0. */ + if (!freq_scale) + freq_scale = 1; + freq = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE; if (freq > CGRAPH_FREQ_MAX) freq = CGRAPH_FREQ_MAX; new_edge = cgraph_create_edge (n, e->callee, call_stmt, count, freq, diff --git a/gcc/cgraphbuild.c b/gcc/cgraphbuild.c index 8423c36b880..0c3bff2f90e 100644 --- a/gcc/cgraphbuild.c +++ b/gcc/cgraphbuild.c @@ -109,7 +109,8 @@ reset_inline_failed (struct cgraph_node *node) int compute_call_stmt_bb_frequency (tree decl, basic_block bb) { - int entry_freq = ENTRY_BLOCK_PTR->frequency; + int entry_freq = ENTRY_BLOCK_PTR_FOR_FUNCTION + (DECL_STRUCT_FUNCTION (decl))->frequency; int freq = bb->frequency; if (profile_status_for_function (DECL_STRUCT_FUNCTION (decl)) == PROFILE_ABSENT) diff --git a/gcc/ipa-inline.c b/gcc/ipa-inline.c index 08088dd3e79..e9d831153fe 100644 --- a/gcc/ipa-inline.c +++ b/gcc/ipa-inline.c @@ -207,6 +207,29 @@ cgraph_estimate_size_after_inlining (int times, struct cgraph_node *to, return size; } +/* Scale frequency of NODE edges by FREQ_SCALE and increase loop nest + by NEST. */ + +static void +update_noncloned_frequencies (struct cgraph_node *node, + int freq_scale, int nest) +{ + struct cgraph_edge *e; + + /* We do not want to ignore high loop nest after freq drops to 0. */ + if (!freq_scale) + freq_scale = 1; + for (e = node->callees; e; e = e->next_callee) + { + e->loop_nest += nest; + e->frequency = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE; + if (e->frequency > CGRAPH_FREQ_MAX) + e->frequency = CGRAPH_FREQ_MAX; + if (!e->inline_failed) + update_noncloned_frequencies (e->callee, freq_scale, nest); + } +} + /* E is expected to be an edge being inlined. Clone destination node of the edge and redirect it to the new clone. DUPLICATE is used for bookkeeping on whether we are actually creating new @@ -234,6 +257,7 @@ cgraph_clone_inlined_nodes (struct cgraph_edge *e, bool duplicate, } duplicate = false; e->callee->local.externally_visible = false; + update_noncloned_frequencies (e->callee, e->frequency, e->loop_nest); } else { diff --git a/gcc/tree-inline.c b/gcc/tree-inline.c index 629ccfb524f..34a8e9d8d07 100644 --- a/gcc/tree-inline.c +++ b/gcc/tree-inline.c @@ -1472,6 +1472,7 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale, gimple_stmt_iterator gsi, copy_gsi, seq_gsi; basic_block copy_basic_block; tree decl; + gcov_type freq; /* create_basic_block() will append every new block to basic_block_info automatically. */ @@ -1481,11 +1482,12 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale, /* We are going to rebuild frequencies from scratch. These values have just small importance to drive canonicalize_loop_headers. */ - copy_basic_block->frequency = ((gcov_type)bb->frequency - * frequency_scale / REG_BR_PROB_BASE); + freq = ((gcov_type)bb->frequency * frequency_scale / REG_BR_PROB_BASE); - if (copy_basic_block->frequency > BB_FREQ_MAX) - copy_basic_block->frequency = BB_FREQ_MAX; + /* We recompute frequencies after inlining, so this is quite safe. */ + if (freq > BB_FREQ_MAX) + freq = BB_FREQ_MAX; + copy_basic_block->frequency = freq; copy_gsi = gsi_start_bb (copy_basic_block); @@ -1631,10 +1633,34 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale, case CB_CGE_DUPLICATE: edge = cgraph_edge (id->src_node, orig_stmt); if (edge) - edge = cgraph_clone_edge (edge, id->dst_node, stmt, - gimple_uid (stmt), - REG_BR_PROB_BASE, 1, - edge->frequency, true); + { + int edge_freq = edge->frequency; + edge = cgraph_clone_edge (edge, id->dst_node, stmt, + gimple_uid (stmt), + REG_BR_PROB_BASE, CGRAPH_FREQ_BASE, + edge->frequency, true); + /* We could also just rescale the frequency, but + doing so would introduce roundoff errors and make + verifier unhappy. */ + edge->frequency + = compute_call_stmt_bb_frequency (id->dst_node->decl, + copy_basic_block); + if (dump_file + && profile_status_for_function (cfun) != PROFILE_ABSENT + && (edge_freq > edge->frequency + 10 + || edge_freq < edge->frequency - 10)) + { + fprintf (dump_file, "Edge frequency estimated by " + "cgraph %i diverge from inliner's estimate %i\n", + edge_freq, + edge->frequency); + fprintf (dump_file, + "Orig bb: %i, orig bb freq %i, new bb freq %i\n", + bb->index, + bb->frequency, + copy_basic_block->frequency); + } + } break; case CB_CGE_MOVE_CLONES: @@ -1674,7 +1700,8 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale, if (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES) cgraph_create_edge_including_clones (id->dst_node, dest, stmt, bb->count, - compute_call_stmt_bb_frequency (id->dst_node->decl, bb), + compute_call_stmt_bb_frequency (id->dst_node->decl, + copy_basic_block), bb->loop_depth, CIF_ORIGINALLY_INDIRECT_CALL); else cgraph_create_edge (id->dst_node, dest, stmt, @@ -1948,24 +1975,16 @@ remap_decl_1 (tree decl, void *data) NEW_FNDECL to be build. CALLEE_FNDECL is the original */ static void -initialize_cfun (tree new_fndecl, tree callee_fndecl, gcov_type count, - int frequency) +initialize_cfun (tree new_fndecl, tree callee_fndecl, gcov_type count) { struct function *src_cfun = DECL_STRUCT_FUNCTION (callee_fndecl); - gcov_type count_scale, frequency_scale; + gcov_type count_scale; if (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count) count_scale = (REG_BR_PROB_BASE * count / ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count); else - count_scale = 1; - - if (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency) - frequency_scale = (REG_BR_PROB_BASE * frequency - / - ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency); - else - frequency_scale = count_scale; + count_scale = REG_BR_PROB_BASE; /* Register specific tree functions. */ gimple_register_cfg_hooks (); @@ -1998,18 +2017,17 @@ initialize_cfun (tree new_fndecl, tree callee_fndecl, gcov_type count, init_empty_tree_cfg (); + profile_status_for_function (cfun) = profile_status_for_function (src_cfun); ENTRY_BLOCK_PTR->count = (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count * count_scale / REG_BR_PROB_BASE); - ENTRY_BLOCK_PTR->frequency = - (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency * - frequency_scale / REG_BR_PROB_BASE); + ENTRY_BLOCK_PTR->frequency + = ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency; EXIT_BLOCK_PTR->count = (EXIT_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count * count_scale / REG_BR_PROB_BASE); EXIT_BLOCK_PTR->frequency = - (EXIT_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency * - frequency_scale / REG_BR_PROB_BASE); + EXIT_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency; if (src_cfun->eh) init_eh_for_function (); @@ -2026,7 +2044,7 @@ initialize_cfun (tree new_fndecl, tree callee_fndecl, gcov_type count, another function. Walks FN via CFG, returns new fndecl. */ static tree -copy_cfg_body (copy_body_data * id, gcov_type count, int frequency, +copy_cfg_body (copy_body_data * id, gcov_type count, int frequency_scale, basic_block entry_block_map, basic_block exit_block_map) { tree callee_fndecl = id->src_fn; @@ -2035,21 +2053,14 @@ copy_cfg_body (copy_body_data * id, gcov_type count, int frequency, struct function *cfun_to_copy; basic_block bb; tree new_fndecl = NULL; - gcov_type count_scale, frequency_scale; + gcov_type count_scale; int last; if (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count) count_scale = (REG_BR_PROB_BASE * count / ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count); else - count_scale = 1; - - if (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency) - frequency_scale = (REG_BR_PROB_BASE * frequency - / - ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency); - else - frequency_scale = count_scale; + count_scale = REG_BR_PROB_BASE; /* Register specific tree functions. */ gimple_register_cfg_hooks (); @@ -2204,7 +2215,7 @@ copy_tree_body (copy_body_data *id) another function. */ static tree -copy_body (copy_body_data *id, gcov_type count, int frequency, +copy_body (copy_body_data *id, gcov_type count, int frequency_scale, basic_block entry_block_map, basic_block exit_block_map) { tree fndecl = id->src_fn; @@ -2212,7 +2223,7 @@ copy_body (copy_body_data *id, gcov_type count, int frequency, /* If this body has a CFG, walk CFG and copy. */ gcc_assert (ENTRY_BLOCK_PTR_FOR_FUNCTION (DECL_STRUCT_FUNCTION (fndecl))); - body = copy_cfg_body (id, count, frequency, entry_block_map, exit_block_map); + body = copy_cfg_body (id, count, frequency_scale, entry_block_map, exit_block_map); copy_debug_stmts (id); return body; @@ -3732,12 +3743,23 @@ expand_call_inline (basic_block bb, gimple stmt, copy_body_data *id) cfun->local_decls); } + if (dump_file && (dump_flags & TDF_DETAILS)) + { + fprintf (dump_file, "Inlining "); + print_generic_expr (dump_file, id->src_fn, 0); + fprintf (dump_file, " to "); + print_generic_expr (dump_file, id->dst_fn, 0); + fprintf (dump_file, " with frequency %i\n", cg_edge->frequency); + } + /* This is it. Duplicate the callee body. Assume callee is pre-gimplified. Note that we must not alter the caller function in any way before this point, as this CALL_EXPR may be a self-referential call; if we're calling ourselves, we need to duplicate our body before altering anything. */ - copy_body (id, bb->count, bb->frequency, bb, return_block); + copy_body (id, bb->count, + cg_edge->frequency * REG_BR_PROB_BASE / CGRAPH_FREQ_BASE, + bb, return_block); /* Reset the escaped and callused solutions. */ if (cfun->gimple_df) @@ -4732,30 +4754,6 @@ delete_unreachable_blocks_update_callgraph (copy_body_data *id) if (changed) tidy_fallthru_edges (); -#ifdef ENABLE_CHECKING0 - verify_cgraph_node (id->dst_node); - if (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES - && id->dst_node->clones) - { - struct cgraph_node *node; - for (node = id->dst_node->clones; node != id->dst_node;) - { - verify_cgraph_node (node); - - if (node->clones) - node = node->clones; - else if (node->next_sibling_clone) - node = node->next_sibling_clone; - else - { - while (node != id->dst_node && !node->next_sibling_clone) - node = node->clone_of; - if (node != id->dst_node) - node = node->next_sibling_clone; - } - } - } -#endif return changed; } @@ -4876,8 +4874,7 @@ tree_function_versioning (tree old_decl, tree new_decl, old_entry_block = ENTRY_BLOCK_PTR_FOR_FUNCTION (DECL_STRUCT_FUNCTION (old_decl)); initialize_cfun (new_decl, old_decl, - old_entry_block->count, - old_entry_block->frequency); + old_entry_block->count); push_cfun (DECL_STRUCT_FUNCTION (new_decl)); /* Copy the function's static chain. */ @@ -4947,7 +4944,7 @@ tree_function_versioning (tree old_decl, tree new_decl, } /* Copy the Function's body. */ - copy_body (&id, old_entry_block->count, old_entry_block->frequency, + copy_body (&id, old_entry_block->count, REG_BR_PROB_BASE, ENTRY_BLOCK_PTR, EXIT_BLOCK_PTR); if (DECL_RESULT (old_decl) != NULL_TREE)