diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 9fc63c540ec..9f1a365ed2e 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,28 @@ +2019-06-10 Martin Liska + + * gcov-io.h (GCOV_DISK_SINGLE_VALUES): New. + (GCOV_SINGLE_VALUE_COUNTERS): Likewise. + * ipa-profile.c (ipa_profile_generate_summary): + Use get_most_common_single_value. + * tree-profile.c (gimple_init_gcov_profiler): + Instrument with __gcov_one_value_profiler_v2 + and __gcov_indirect_call_profiler_v4. + * value-prof.c (dump_histogram_value): + Print all values for HIST_TYPE_SINGLE_VALUE. + (stream_out_histogram_value): Update assert for + N values. + (stream_in_histogram_value): Set number of + counters for HIST_TYPE_SINGLE_VALUE. + (get_most_common_single_value): New. + (gimple_divmod_fixed_value_transform): + Use get_most_common_single_value. + (gimple_ic_transform): Likewise. + (gimple_stringops_transform): Likewise. + (gimple_find_values_to_profile): Set number + of counters for HIST_TYPE_SINGLE_VALUE. + * value-prof.h (get_most_common_single_value): + New. + 2019-06-10 Martin Liska * hash-map.h: Pass default value to hash_table ctor. diff --git a/gcc/gcov-io.h b/gcc/gcov-io.h index 69c9a73dba8..0f2905c17ec 100644 --- a/gcc/gcov-io.h +++ b/gcc/gcov-io.h @@ -266,6 +266,13 @@ GCOV_COUNTERS #define GCOV_N_VALUE_COUNTERS \ (GCOV_LAST_VALUE_COUNTER - GCOV_FIRST_VALUE_COUNTER + 1) +/* Number of single value histogram values that live + on disk representation. */ +#define GCOV_DISK_SINGLE_VALUES 4 + +/* Total number of single value counters. */ +#define GCOV_SINGLE_VALUE_COUNTERS (2 * GCOV_DISK_SINGLE_VALUES + 1) + /* Convert a counter index to a tag. */ #define GCOV_TAG_FOR_COUNTER(COUNT) \ (GCOV_TAG_COUNTER_BASE + ((gcov_unsigned_t)(COUNT) << 17)) diff --git a/gcc/ipa-profile.c b/gcc/ipa-profile.c index de9563d808c..c80ea7a9b95 100644 --- a/gcc/ipa-profile.c +++ b/gcc/ipa-profile.c @@ -191,17 +191,17 @@ ipa_profile_generate_summary (void) takes away bad histograms. */ if (h) { - /* counter 0 is target, counter 1 is number of execution we called target, - counter 2 is total number of executions. */ - if (h->hvalue.counters[2]) + gcov_type val, count, all; + if (get_most_common_single_value (NULL, "indirect call", + h, &val, &count, &all)) { struct cgraph_edge * e = node->get_edge (stmt); if (e && !e->indirect_unknown_callee) continue; - e->indirect_info->common_target_id - = h->hvalue.counters [0]; + + e->indirect_info->common_target_id = val; e->indirect_info->common_target_probability - = GCOV_COMPUTE_SCALE (h->hvalue.counters [1], h->hvalue.counters [2]); + = GCOV_COMPUTE_SCALE (count, all); if (e->indirect_info->common_target_probability > REG_BR_PROB_BASE) { if (dump_file) diff --git a/gcc/tree-profile.c b/gcc/tree-profile.c index f2cf4047579..5ca4c3e80b6 100644 --- a/gcc/tree-profile.c +++ b/gcc/tree-profile.c @@ -165,10 +165,10 @@ gimple_init_gcov_profiler (void) = build_function_type_list (void_type_node, gcov_type_ptr, gcov_type_node, NULL_TREE); - fn_name = concat ("__gcov_one_value_profiler", fn_suffix, NULL); + fn_name = concat ("__gcov_one_value_profiler_v2", fn_suffix, NULL); tree_one_value_profiler_fn = build_fn_decl (fn_name, one_value_profiler_fn_type); - free (CONST_CAST (char *, fn_name)); + TREE_NOTHROW (tree_one_value_profiler_fn) = 1; DECL_ATTRIBUTES (tree_one_value_profiler_fn) = tree_cons (get_identifier ("leaf"), NULL, @@ -182,7 +182,7 @@ gimple_init_gcov_profiler (void) gcov_type_node, ptr_type_node, NULL_TREE); - profiler_fn_name = "__gcov_indirect_call_profiler_v3"; + profiler_fn_name = "__gcov_indirect_call_profiler_v4"; tree_indirect_call_profiler_fn = build_fn_decl (profiler_fn_name, ic_profiler_fn_type); diff --git a/gcc/value-prof.c b/gcc/value-prof.c index 1e14e532070..b95bf85270b 100644 --- a/gcc/value-prof.c +++ b/gcc/value-prof.c @@ -259,15 +259,22 @@ dump_histogram_value (FILE *dump_file, histogram_value hist) break; case HIST_TYPE_SINGLE_VALUE: - fprintf (dump_file, "Single value "); + case HIST_TYPE_INDIR_CALL: + fprintf (dump_file, + (hist->type == HIST_TYPE_SINGLE_VALUE + ? "Single value counter " : "Indirect call counter")); if (hist->hvalue.counters) { - fprintf (dump_file, "value:%" PRId64 - " match:%" PRId64 - " wrong:%" PRId64, - (int64_t) hist->hvalue.counters[0], - (int64_t) hist->hvalue.counters[1], - (int64_t) hist->hvalue.counters[2]); + fprintf (dump_file, "all: %" PRId64 ", values: ", + (int64_t) hist->hvalue.counters[0]); + for (unsigned i = 0; i < GCOV_DISK_SINGLE_VALUES; i++) + { + fprintf (dump_file, "[%" PRId64 ":%" PRId64 "]", + (int64_t) hist->hvalue.counters[2 * i + 1], + (int64_t) hist->hvalue.counters[2 * i + 2]); + if (i != GCOV_DISK_SINGLE_VALUES - 1) + fprintf (dump_file, ", "); + } } fprintf (dump_file, ".\n"); break; @@ -294,19 +301,6 @@ dump_histogram_value (FILE *dump_file, histogram_value hist) fprintf (dump_file, ".\n"); break; - case HIST_TYPE_INDIR_CALL: - fprintf (dump_file, "Indirect call "); - if (hist->hvalue.counters) - { - fprintf (dump_file, "value:%" PRId64 - " match:%" PRId64 - " all:%" PRId64, - (int64_t) hist->hvalue.counters[0], - (int64_t) hist->hvalue.counters[1], - (int64_t) hist->hvalue.counters[2]); - } - fprintf (dump_file, ".\n"); - break; case HIST_TYPE_TIME_PROFILE: fprintf (dump_file, "Time profile "); if (hist->hvalue.counters) @@ -347,7 +341,7 @@ stream_out_histogram_value (struct output_block *ob, histogram_value hist) /* When user uses an unsigned type with a big value, constant converted to gcov_type (a signed type) can be negative. */ gcov_type value = hist->hvalue.counters[i]; - if (hist->type == HIST_TYPE_SINGLE_VALUE && i == 0) + if (hist->type == HIST_TYPE_SINGLE_VALUE && (i > 0 && ((i - 1) % 2) == 0)) ; else gcc_assert (value >= 0); @@ -392,7 +386,7 @@ stream_in_histogram_value (struct lto_input_block *ib, gimple *stmt) case HIST_TYPE_SINGLE_VALUE: case HIST_TYPE_INDIR_CALL: - ncounters = 3; + ncounters = GCOV_SINGLE_VALUE_COUNTERS; break; case HIST_TYPE_IOR: @@ -729,6 +723,48 @@ gimple_divmod_fixed_value (gassign *stmt, tree value, profile_probability prob, return tmp2; } +/* Return most common value of SINGLE_VALUE histogram. If + there's a unique value, return true and set VALUE and COUNT + arguments. */ + +bool +get_most_common_single_value (gimple *stmt, const char *counter_type, + histogram_value hist, + gcov_type *value, gcov_type *count, + gcov_type *all) +{ + if (hist->hvalue.counters[2] == -1) + return false; + + *count = 0; + *value = 0; + + gcov_type read_all = hist->hvalue.counters[0]; + + for (unsigned i = 0; i < GCOV_DISK_SINGLE_VALUES; i++) + { + gcov_type v = hist->hvalue.counters[2 * i + 1]; + gcov_type c = hist->hvalue.counters[2 * i + 2]; + + /* Indirect calls can't be vereified. */ + if (stmt && check_counter (stmt, counter_type, &c, &read_all, + gimple_bb (stmt)->count)) + return false; + + *all = read_all; + + if (c > *count) + { + *value = v; + *count = c; + } + else if (c == *count && v > *value) + *value = v; + } + + return true; +} + /* Do transform 1) on INSN if applicable. */ static bool @@ -758,23 +794,19 @@ gimple_divmod_fixed_value_transform (gimple_stmt_iterator *si) if (!histogram) return false; + if (!get_most_common_single_value (stmt, "divmod", histogram, &val, &count, + &all)) + return false; + value = histogram->hvalue.value; - val = histogram->hvalue.counters[0]; - count = histogram->hvalue.counters[1]; - all = histogram->hvalue.counters[2]; gimple_remove_histogram_value (cfun, stmt, histogram); - /* We require that count is at least half of all; this means - that for the transformation to fire the value must be constant - at least 50% of time (and 75% gives the guarantee of usage). */ + /* We require that count is at least half of all. */ if (simple_cst_equal (gimple_assign_rhs2 (stmt), value) != 1 || 2 * count < all || optimize_bb_for_size_p (gimple_bb (stmt))) return false; - if (check_counter (stmt, "value", &count, &all, gimple_bb (stmt)->count)) - return false; - /* Compute probability of taking the optimal path. */ if (all > 0) prob = profile_probability::probability_in_gcov_type (count, all); @@ -1401,7 +1433,7 @@ gimple_ic_transform (gimple_stmt_iterator *gsi) { gcall *stmt; histogram_value histogram; - gcov_type val, count, all, bb_all; + gcov_type val, count, all; struct cgraph_node *direct_call; stmt = dyn_cast (gsi_stmt (*gsi)); @@ -1418,21 +1450,9 @@ gimple_ic_transform (gimple_stmt_iterator *gsi) if (!histogram) return false; - val = histogram->hvalue.counters [0]; - count = histogram->hvalue.counters [1]; - all = histogram->hvalue.counters [2]; - - bb_all = gimple_bb (stmt)->count.ipa ().to_gcov_type (); - /* The order of CHECK_COUNTER calls is important - - since check_counter can correct the third parameter - and we want to make count <= all <= bb_all. */ - if (check_counter (stmt, "ic", &all, &bb_all, gimple_bb (stmt)->count) - || check_counter (stmt, "ic", &count, &all, - profile_count::from_gcov_type (all))) - { - gimple_remove_histogram_value (cfun, stmt, histogram); - return false; - } + if (!get_most_common_single_value (NULL, "indirect call", histogram, &val, + &count, &all)) + return false; if (4 * count <= 3 * all) return false; @@ -1644,19 +1664,19 @@ gimple_stringops_transform (gimple_stmt_iterator *gsi) if (TREE_CODE (blck_size) == INTEGER_CST) return false; - histogram = gimple_histogram_value_of_type (cfun, stmt, HIST_TYPE_SINGLE_VALUE); + histogram = gimple_histogram_value_of_type (cfun, stmt, + HIST_TYPE_SINGLE_VALUE); if (!histogram) return false; - val = histogram->hvalue.counters[0]; - count = histogram->hvalue.counters[1]; - all = histogram->hvalue.counters[2]; + if (!get_most_common_single_value (stmt, "stringops", histogram, &val, + &count, &all)) + return false; + gimple_remove_histogram_value (cfun, stmt, histogram); - /* We require that count is at least half of all; this means - that for the transformation to fire the value must be constant - at least 80% of time. */ - if ((6 * count / 5) < all || optimize_bb_for_size_p (gimple_bb (stmt))) + /* We require that count is at least half of all. */ + if (2 * count < all || optimize_bb_for_size_p (gimple_bb (stmt))) return false; if (check_counter (stmt, "value", &count, &all, gimple_bb (stmt)->count)) return false; @@ -1928,11 +1948,11 @@ gimple_find_values_to_profile (histogram_values *values) break; case HIST_TYPE_SINGLE_VALUE: - hist->n_counters = 3; + hist->n_counters = GCOV_SINGLE_VALUE_COUNTERS; break; - case HIST_TYPE_INDIR_CALL: - hist->n_counters = 3; + case HIST_TYPE_INDIR_CALL: + hist->n_counters = GCOV_SINGLE_VALUE_COUNTERS; break; case HIST_TYPE_TIME_PROFILE: diff --git a/gcc/value-prof.h b/gcc/value-prof.h index a54024b48de..25b03f7591a 100644 --- a/gcc/value-prof.h +++ b/gcc/value-prof.h @@ -90,6 +90,10 @@ void free_histograms (function *); void stringop_block_profile (gimple *, unsigned int *, HOST_WIDE_INT *); gcall *gimple_ic (gcall *, struct cgraph_node *, profile_probability); bool check_ic_target (gcall *, struct cgraph_node *); +bool get_most_common_single_value (gimple *stmt, const char *counter_type, + histogram_value hist, + gcov_type *value, gcov_type *count, + gcov_type *all); /* In tree-profile.c. */ diff --git a/libgcc/ChangeLog b/libgcc/ChangeLog index 7ef347a1834..669041e505f 100644 --- a/libgcc/ChangeLog +++ b/libgcc/ChangeLog @@ -1,3 +1,27 @@ +2019-06-10 Martin Liska + + * Makefile.in: Add __gcov_one_value_profiler_v2, + __gcov_one_value_profiler_v2_atomic and + __gcov_indirect_call_profiler_v4. + * libgcov-merge.c (__gcov_merge_single): Change + function signature. + (merge_single_value_set): New. + * libgcov-profiler.c (__gcov_one_value_profiler_body): + Update functionality. + (__gcov_one_value_profiler): Remove. + (__gcov_one_value_profiler_v2): ... this. + (__gcov_one_value_profiler_atomic): Rename to ... + (__gcov_one_value_profiler_v2_atomic): this. + (__gcov_indirect_call_profiler_v3): Rename to ... + (__gcov_indirect_call_profiler_v4): ... this. + * libgcov.h (__gcov_one_value_profiler): Remove. + (__gcov_one_value_profiler_atomic): Remove. + (__gcov_one_value_profiler_v2_atomic): New. + (__gcov_indirect_call_profiler_v3): Remove. + (__gcov_one_value_profiler_v2): New. + (__gcov_indirect_call_profiler_v4): New. + (gcov_get_counter_ignore_scaling): New function. + 2019-06-07 Martin Liska * Makefile.in: Remove usage of diff --git a/libgcc/Makefile.in b/libgcc/Makefile.in index fb77881145e..33b83809cfc 100644 --- a/libgcc/Makefile.in +++ b/libgcc/Makefile.in @@ -893,13 +893,13 @@ LIBGCOV_PROFILER = _gcov_interval_profiler \ _gcov_interval_profiler_atomic \ _gcov_pow2_profiler \ _gcov_pow2_profiler_atomic \ - _gcov_one_value_profiler \ - _gcov_one_value_profiler_atomic \ + _gcov_one_value_profiler_v2 \ + _gcov_one_value_profiler_v2_atomic \ _gcov_average_profiler \ _gcov_average_profiler_atomic \ _gcov_ior_profiler \ _gcov_ior_profiler_atomic \ - _gcov_indirect_call_profiler_v3 \ + _gcov_indirect_call_profiler_v4 \ _gcov_time_profiler LIBGCOV_INTERFACE = _gcov_dump _gcov_flush _gcov_fork \ _gcov_execl _gcov_execlp \ diff --git a/libgcc/libgcov-merge.c b/libgcc/libgcov-merge.c index 702a69f8349..42416341b4d 100644 --- a/libgcc/libgcov-merge.c +++ b/libgcc/libgcov-merge.c @@ -34,8 +34,9 @@ void __gcov_merge_add (gcov_type *counters __attribute__ ((unused)), #endif #ifdef L_gcov_merge_single -void __gcov_merge_single (gcov_type *counters __attribute__ ((unused)), - unsigned n_counters __attribute__ ((unused))) {} +void __gcov_merge_single (gcov_type *counters __attribute__ ((unused))) +{ +} #endif #else @@ -85,40 +86,72 @@ __gcov_merge_time_profile (gcov_type *counters, unsigned n_counters) #endif /* L_gcov_merge_time_profile */ #ifdef L_gcov_merge_single + +static void +merge_single_value_set (gcov_type *counters) +{ + unsigned j; + gcov_type value, counter; + + /* First value is number of total executions of the profiler. */ + gcov_type all = gcov_get_counter_ignore_scaling (-1); + counters[0] += all; + ++counters; + + for (unsigned i = 0; i < GCOV_DISK_SINGLE_VALUES; i++) + { + value = gcov_get_counter_target (); + counter = gcov_get_counter_ignore_scaling (-1); + + if (counter == -1) + { + counters[1] = -1; + /* We can't return as we need to read all counters. */ + continue; + } + else if (counter == 0 || counters[1] == -1) + { + /* We can't return as we need to read all counters. */ + continue; + } + + for (j = 0; j < GCOV_DISK_SINGLE_VALUES; j++) + { + if (counters[2 * j] == value) + { + counters[2 * j + 1] += counter; + break; + } + else if (counters[2 * j + 1] == 0) + { + counters[2 * j] = value; + counters[2 * j + 1] = counter; + break; + } + } + + /* We haven't found a free slot for the value, mark overflow. */ + if (j == GCOV_DISK_SINGLE_VALUES) + counters[1] = -1; + } +} + /* The profile merging function for choosing the most common value. It is given an array COUNTERS of N_COUNTERS old counters and it reads the same number of counters from the gcov file. The counters - are split into 3-tuples where the members of the tuple have + are split into pairs where the members of the tuple have meanings: -- the stored candidate on the most common value of the measured entity -- counter - -- total number of evaluations of the value */ + */ void __gcov_merge_single (gcov_type *counters, unsigned n_counters) { - unsigned i, n_measures; - gcov_type value, counter, all; + gcc_assert (!(n_counters % GCOV_SINGLE_VALUE_COUNTERS)); - gcc_assert (!(n_counters % 3)); - n_measures = n_counters / 3; - for (i = 0; i < n_measures; i++, counters += 3) - { - value = gcov_get_counter_target (); - counter = gcov_get_counter (); - all = gcov_get_counter (); - - if (counters[0] == value) - counters[1] += counter; - else if (counter > counters[1]) - { - counters[0] = value; - counters[1] = counter - counters[1]; - } - else - counters[1] -= counter; - counters[2] += all; - } + for (unsigned i = 0; i < (n_counters / GCOV_SINGLE_VALUE_COUNTERS); i++) + merge_single_value_set (counters + (i * GCOV_SINGLE_VALUE_COUNTERS)); } #endif /* L_gcov_merge_single */ diff --git a/libgcc/libgcov-profiler.c b/libgcc/libgcov-profiler.c index 40f0858a174..9ba65b90df3 100644 --- a/libgcc/libgcov-profiler.c +++ b/libgcc/libgcov-profiler.c @@ -112,40 +112,37 @@ __gcov_pow2_profiler_atomic (gcov_type *counters, gcov_type value) COUNTERS[1] is decremented. Otherwise COUNTERS[1] is set to one and VALUE is stored to COUNTERS[0]. This algorithm guarantees that if this function is called more than 50% of the time with one value, this value - will be in COUNTERS[0] in the end. - - In any case, COUNTERS[2] is incremented. If USE_ATOMIC is set to 1, - COUNTERS[2] is updated with an atomic instruction. */ + will be in COUNTERS[0] in the end. */ static inline void __gcov_one_value_profiler_body (gcov_type *counters, gcov_type value, int use_atomic) { - if (value == counters[0]) - counters[1]++; - else if (counters[1] == 0) + if (value == counters[1]) + counters[2]++; + else if (counters[2] == 0) { - counters[1] = 1; - counters[0] = value; + counters[2] = 1; + counters[1] = value; } else - counters[1]--; + counters[2]--; if (use_atomic) - __atomic_fetch_add (&counters[2], 1, __ATOMIC_RELAXED); + __atomic_fetch_add (&counters[0], 1, __ATOMIC_RELAXED); else - counters[2]++; + counters[0]++; } -#ifdef L_gcov_one_value_profiler +#ifdef L_gcov_one_value_profiler_v2 void -__gcov_one_value_profiler (gcov_type *counters, gcov_type value) +__gcov_one_value_profiler_v2 (gcov_type *counters, gcov_type value) { __gcov_one_value_profiler_body (counters, value, 0); } #endif -#if defined(L_gcov_one_value_profiler_atomic) && GCOV_SUPPORTS_ATOMIC +#if defined(L_gcov_one_value_profiler_v2_atomic) && GCOV_SUPPORTS_ATOMIC /* Update one value profilers (COUNTERS) for a given VALUE. @@ -157,13 +154,13 @@ __gcov_one_value_profiler (gcov_type *counters, gcov_type value) https://gcc.gnu.org/ml/gcc-patches/2016-08/msg00024.html. */ void -__gcov_one_value_profiler_atomic (gcov_type *counters, gcov_type value) +__gcov_one_value_profiler_v2_atomic (gcov_type *counters, gcov_type value) { __gcov_one_value_profiler_body (counters, value, 1); } #endif -#ifdef L_gcov_indirect_call_profiler_v3 +#ifdef L_gcov_indirect_call_profiler_v4 /* These two variables are used to actually track caller and callee. Keep them in TLS memory so races are not common (they are written to often). @@ -185,7 +182,7 @@ struct indirect_call_tuple __gcov_indirect_call; /* Tries to determine the most common value among its inputs. */ void -__gcov_indirect_call_profiler_v3 (gcov_type value, void* cur_func) +__gcov_indirect_call_profiler_v4 (gcov_type value, void* cur_func) { /* If the C++ virtual tables contain function descriptors then one function may have multiple descriptors and we need to dereference diff --git a/libgcc/libgcov.h b/libgcc/libgcov.h index b4f1ec576fc..144b4817a37 100644 --- a/libgcc/libgcov.h +++ b/libgcc/libgcov.h @@ -271,9 +271,9 @@ extern void __gcov_interval_profiler_atomic (gcov_type *, gcov_type, int, unsigned); extern void __gcov_pow2_profiler (gcov_type *, gcov_type); extern void __gcov_pow2_profiler_atomic (gcov_type *, gcov_type); -extern void __gcov_one_value_profiler (gcov_type *, gcov_type); -extern void __gcov_one_value_profiler_atomic (gcov_type *, gcov_type); -extern void __gcov_indirect_call_profiler_v3 (gcov_type, void *); +extern void __gcov_one_value_profiler_v2 (gcov_type *, gcov_type); +extern void __gcov_one_value_profiler_v2_atomic (gcov_type *, gcov_type); +extern void __gcov_indirect_call_profiler_v4 (gcov_type, void *); extern void __gcov_time_profiler (gcov_type *); extern void __gcov_time_profiler_atomic (gcov_type *); extern void __gcov_average_profiler (gcov_type *, gcov_type); @@ -324,6 +324,29 @@ gcov_get_counter (void) #endif } +/* Similar function as gcov_get_counter(), but do not scale + when read value is equal to IGNORE_SCALING. */ + +static inline gcov_type +gcov_get_counter_ignore_scaling (gcov_type ignore_scaling) +{ +#ifndef IN_GCOV_TOOL + /* This version is for reading count values in libgcov runtime: + we read from gcda files. */ + + return gcov_read_counter (); +#else + /* This version is for gcov-tool. We read the value from memory and + multiply it by the merge weight. */ + + gcov_type v = gcov_read_counter_mem (); + if (v != ignore_scaling) + v *= gcov_get_merge_weight (); + + return v; +#endif +} + /* Similar function as gcov_get_counter(), but handles target address counters. */