From 61d371eb413e6917f59bea804b3c167e31f88f98 Mon Sep 17 00:00:00 2001 From: Richard Biener Date: Fri, 6 Nov 2015 11:15:40 +0000 Subject: [PATCH] tree-vectorizer.h (struct _bb_vec_info): Add region_begin/end members. 2015-11-06 Richard Biener * tree-vectorizer.h (struct _bb_vec_info): Add region_begin/end members. (vect_stmt_in_region_p): Declare. * tree-vect-slp.c (new_bb_vec_info): Work on a region. (destroy_bb_vec_info): Likewise. (vect_bb_slp_scalar_cost): Use vect_stmt_in_region_p. (vect_get_and_check_slp_defs): Likewise. (vect_slp_analyze_bb_1): Refactor to make it work on sub-BBs. (vect_slp_bb): Likewise. * tree-vect-patterns.c (vect_same_loop_or_bb_p): Implement in terms of vect_stmt_in_region_p. (vect_pattern_recog): Iterate over the BB region. * tree-vect-stmts.c (vect_is_simple_use): Use vect_stmt_in_region_p. * tree-vectorizer.c (vect_stmt_in_region_p): New function. (pass_slp_vectorize::execute): Initialize all stmt UIDs to -1. * config/i386/i386.c: Include gimple-iterator.h. * config/aarch64/aarch64.c: Likewise. * gcc.dg/vect/bb-slp-38.c: New testcase. From-SVN: r229842 --- gcc/ChangeLog | 20 +++ gcc/config/aarch64/aarch64.c | 1 + gcc/config/i386/i386.c | 1 + gcc/testsuite/ChangeLog | 4 + gcc/testsuite/gcc.dg/vect/bb-slp-38.c | 44 ++++++ gcc/tree-vect-patterns.c | 69 ++++------ gcc/tree-vect-slp.c | 186 ++++++++++++++------------ gcc/tree-vect-stmts.c | 7 +- gcc/tree-vectorizer.c | 35 +++++ gcc/tree-vectorizer.h | 3 + 10 files changed, 241 insertions(+), 129 deletions(-) create mode 100644 gcc/testsuite/gcc.dg/vect/bb-slp-38.c diff --git a/gcc/ChangeLog b/gcc/ChangeLog index ae2c5cf4857..0355c9eaafa 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,23 @@ +2015-11-06 Richard Biener + + * tree-vectorizer.h (struct _bb_vec_info): Add region_begin/end + members. + (vect_stmt_in_region_p): Declare. + * tree-vect-slp.c (new_bb_vec_info): Work on a region. + (destroy_bb_vec_info): Likewise. + (vect_bb_slp_scalar_cost): Use vect_stmt_in_region_p. + (vect_get_and_check_slp_defs): Likewise. + (vect_slp_analyze_bb_1): Refactor to make it work on sub-BBs. + (vect_slp_bb): Likewise. + * tree-vect-patterns.c (vect_same_loop_or_bb_p): Implement + in terms of vect_stmt_in_region_p. + (vect_pattern_recog): Iterate over the BB region. + * tree-vect-stmts.c (vect_is_simple_use): Use vect_stmt_in_region_p. + * tree-vectorizer.c (vect_stmt_in_region_p): New function. + (pass_slp_vectorize::execute): Initialize all stmt UIDs to -1. + * config/i386/i386.c: Include gimple-iterator.h. + * config/aarch64/aarch64.c: Likewise. + 2015-11-06 Alexandre Oliva PR rtl-optimization/67753 diff --git a/gcc/config/aarch64/aarch64.c b/gcc/config/aarch64/aarch64.c index 5c8604ffa72..98b13b4b7e2 100644 --- a/gcc/config/aarch64/aarch64.c +++ b/gcc/config/aarch64/aarch64.c @@ -52,6 +52,7 @@ #include "params.h" #include "gimplify.h" #include "dwarf2.h" +#include "gimple-iterator.h" #include "tree-vectorizer.h" #include "aarch64-cost-tables.h" #include "dumpfile.h" diff --git a/gcc/config/i386/i386.c b/gcc/config/i386/i386.c index 2a965f678a0..bb37abaf119 100644 --- a/gcc/config/i386/i386.c +++ b/gcc/config/i386/i386.c @@ -64,6 +64,7 @@ along with GCC; see the file COPYING3. If not see #include "context.h" #include "pass_manager.h" #include "target-globals.h" +#include "gimple-iterator.h" #include "tree-vectorizer.h" #include "shrink-wrap.h" #include "builtins.h" diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog index b0e78e9e639..3d1834dfb81 100644 --- a/gcc/testsuite/ChangeLog +++ b/gcc/testsuite/ChangeLog @@ -1,3 +1,7 @@ +2015-11-06 Richard Biener + + * gcc.dg/vect/bb-slp-38.c: New testcase. + 2015-11-06 Thomas Schwinge * gfortran.dg/goacc/private-3.f95: XFAIL. diff --git a/gcc/testsuite/gcc.dg/vect/bb-slp-38.c b/gcc/testsuite/gcc.dg/vect/bb-slp-38.c new file mode 100644 index 00000000000..59aec54fffd --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/bb-slp-38.c @@ -0,0 +1,44 @@ +/* { dg-require-effective-target vect_int } */ + +#include "tree-vect.h" + +extern void abort (void); + +int a[8], b[8]; +int x; + +void __attribute__((noinline,noclone)) +bar (void) +{ + x = 1; +} + +void __attribute__((noinline,noclone)) +foo(void) +{ + a[0] = b[0]; + a[1] = b[0]; + a[2] = b[3]; + a[3] = b[3]; + bar (); + a[4] = b[4]; + a[5] = b[7]; + a[6] = b[4]; + a[7] = b[7]; +} + +int main() +{ + int i; + check_vect (); + for (i = 0; i < 8; ++i) + b[i] = i; + foo (); + if (a[0] != 0 || a[1] != 0 || a[2] != 3 || a[3] != 3 + || a[4] != 4 || a[5] != 7 || a[6] != 4 || a[7] != 7) + abort (); + return 0; +} + +/* { dg-final { scan-tree-dump "basic block vectorized" "slp2" { target vect_perm } } } */ +/* { dg-final { scan-tree-dump-times "basic block part vectorized" 2 "slp2" { target vect_perm } } } */ diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c index f1de690f89c..d003d335f5d 100644 --- a/gcc/tree-vect-patterns.c +++ b/gcc/tree-vect-patterns.c @@ -107,27 +107,7 @@ static bool vect_same_loop_or_bb_p (gimple *stmt1, gimple *stmt2) { stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt1); - loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo); - bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_vinfo); - - if (!gimple_bb (stmt2)) - return false; - - if (loop_vinfo) - { - struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); - if (!flow_bb_inside_loop_p (loop, gimple_bb (stmt2))) - return false; - } - else - { - if (gimple_bb (stmt2) != BB_VINFO_BB (bb_vinfo) - || gimple_code (stmt2) == GIMPLE_PHI) - return false; - } - - gcc_assert (vinfo_for_stmt (stmt2)); - return true; + return vect_stmt_in_region_p (stmt_vinfo->vinfo, stmt2); } /* If the LHS of DEF_STMT has a single use, and that statement is @@ -3611,33 +3591,42 @@ vect_pattern_recog (vec_info *vinfo) loop = LOOP_VINFO_LOOP (loop_vinfo); bbs = LOOP_VINFO_BBS (loop_vinfo); nbbs = loop->num_nodes; + + /* Scan through the loop stmts, applying the pattern recognition + functions starting at each stmt visited: */ + for (i = 0; i < nbbs; i++) + { + basic_block bb = bbs[i]; + for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si)) + { + /* Scan over all generic vect_recog_xxx_pattern functions. */ + for (j = 0; j < NUM_PATTERNS; j++) + { + vect_recog_func = vect_vect_recog_func_ptrs[j]; + vect_pattern_recog_1 (vect_recog_func, si, + &stmts_to_replace); + } + } + } } else { - bbs = &as_a (vinfo)->bb; - nbbs = 1; - } - - /* Scan through the loop stmts, applying the pattern recognition - functions starting at each stmt visited: */ - for (i = 0; i < nbbs; i++) - { - basic_block bb = bbs[i]; - for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si)) - { - if (is_a (vinfo) - && (stmt = gsi_stmt (si)) + bb_vec_info bb_vinfo = as_a (vinfo); + for (si = bb_vinfo->region_begin; + gsi_stmt (si) != gsi_stmt (bb_vinfo->region_end); gsi_next (&si)) + { + if ((stmt = gsi_stmt (si)) && vinfo_for_stmt (stmt) && !STMT_VINFO_VECTORIZABLE (vinfo_for_stmt (stmt))) - continue; + continue; - /* Scan over all generic vect_recog_xxx_pattern functions. */ - for (j = 0; j < NUM_PATTERNS; j++) - { + /* Scan over all generic vect_recog_xxx_pattern functions. */ + for (j = 0; j < NUM_PATTERNS; j++) + { vect_recog_func = vect_vect_recog_func_ptrs[j]; vect_pattern_recog_1 (vect_recog_func, si, &stmts_to_replace); - } - } + } + } } } diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c index 35808eb9c8a..cfdfc2936a1 100644 --- a/gcc/tree-vect-slp.c +++ b/gcc/tree-vect-slp.c @@ -209,7 +209,6 @@ vect_get_and_check_slp_defs (vec_info *vinfo, unsigned int i, number_of_oprnds; gimple *def_stmt; enum vect_def_type dt = vect_uninitialized_def; - struct loop *loop = NULL; bool pattern = false; slp_oprnd_info oprnd_info; int first_op_idx = 1; @@ -218,9 +217,6 @@ vect_get_and_check_slp_defs (vec_info *vinfo, bool first = stmt_num == 0; bool second = stmt_num == 1; - if (is_a (vinfo)) - loop = LOOP_VINFO_LOOP (as_a (vinfo)); - if (is_gimple_call (stmt)) { number_of_oprnds = gimple_call_num_args (stmt); @@ -276,11 +272,7 @@ again: from the pattern. Check that all the stmts of the node are in the pattern. */ if (def_stmt && gimple_bb (def_stmt) - && ((is_a (vinfo) - && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))) - || (is_a (vinfo) - && gimple_bb (def_stmt) == as_a (vinfo)->bb - && gimple_code (def_stmt) != GIMPLE_PHI)) + && vect_stmt_in_region_p (vinfo, def_stmt) && vinfo_for_stmt (def_stmt) && STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (def_stmt)) && !STMT_VINFO_RELEVANT (vinfo_for_stmt (def_stmt)) @@ -2076,16 +2068,21 @@ vect_detect_hybrid_slp (loop_vec_info loop_vinfo) stmt_vec_info structs for all the stmts in it. */ static bb_vec_info -new_bb_vec_info (basic_block bb) +new_bb_vec_info (gimple_stmt_iterator region_begin, + gimple_stmt_iterator region_end) { + basic_block bb = gsi_bb (region_begin); bb_vec_info res = NULL; gimple_stmt_iterator gsi; res = (bb_vec_info) xcalloc (1, sizeof (struct _bb_vec_info)); res->kind = vec_info::bb; BB_VINFO_BB (res) = bb; + res->region_begin = region_begin; + res->region_end = region_end; - for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) + for (gsi = region_begin; gsi_stmt (gsi) != gsi_stmt (region_end); + gsi_next (&gsi)) { gimple *stmt = gsi_stmt (gsi); gimple_set_uid (stmt, 0); @@ -2118,7 +2115,8 @@ destroy_bb_vec_info (bb_vec_info bb_vinfo) bb = BB_VINFO_BB (bb_vinfo); - for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si)) + for (si = bb_vinfo->region_begin; + gsi_stmt (si) != gsi_stmt (bb_vinfo->region_end); gsi_next (&si)) { gimple *stmt = gsi_stmt (si); stmt_vec_info stmt_info = vinfo_for_stmt (stmt); @@ -2126,6 +2124,9 @@ destroy_bb_vec_info (bb_vec_info bb_vinfo) if (stmt_info) /* Free stmt_vec_info. */ free_stmt_vec_info (stmt); + + /* Reset region marker. */ + gimple_set_uid (stmt, -1); } vect_destroy_datarefs (bb_vinfo); @@ -2247,8 +2248,8 @@ vect_bb_slp_scalar_cost (basic_block bb, gimple *use_stmt; FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, DEF_FROM_PTR (def_p)) if (!is_gimple_debug (use_stmt) - && (gimple_code (use_stmt) == GIMPLE_PHI - || gimple_bb (use_stmt) != bb + && (! vect_stmt_in_region_p (vinfo_for_stmt (stmt)->vinfo, + use_stmt) || !STMT_VINFO_VECTORIZABLE (vinfo_for_stmt (use_stmt)))) { (*life)[i] = true; @@ -2327,40 +2328,31 @@ vect_bb_vectorization_profitable_p (bb_vec_info bb_vinfo) /* Check if the basic block can be vectorized. */ static bb_vec_info -vect_slp_analyze_bb_1 (basic_block bb) +vect_slp_analyze_bb_1 (gimple_stmt_iterator region_begin, + gimple_stmt_iterator region_end, + vec datarefs, int n_stmts) { bb_vec_info bb_vinfo; vec slp_instances; slp_instance instance; int i; int min_vf = 2; - unsigned n_stmts = 0; - bb_vinfo = new_bb_vec_info (bb); + if (n_stmts > PARAM_VALUE (PARAM_SLP_MAX_INSNS_IN_BB)) + { + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "not vectorized: too many instructions in " + "basic block.\n"); + free_data_refs (datarefs); + return NULL; + } + + bb_vinfo = new_bb_vec_info (region_begin, region_end); if (!bb_vinfo) return NULL; - /* Gather all data references in the basic-block. */ - - for (gimple_stmt_iterator gsi = gsi_start_bb (bb); - !gsi_end_p (gsi); gsi_next (&gsi)) - { - gimple *stmt = gsi_stmt (gsi); - if (is_gimple_debug (stmt)) - continue; - ++n_stmts; - if (!find_data_references_in_stmt (NULL, stmt, - &BB_VINFO_DATAREFS (bb_vinfo))) - { - /* Mark the rest of the basic-block as unvectorizable. */ - for (; !gsi_end_p (gsi); gsi_next (&gsi)) - { - stmt = gsi_stmt (gsi); - STMT_VINFO_VECTORIZABLE (vinfo_for_stmt (stmt)) = false; - } - break; - } - } + BB_VINFO_DATAREFS (bb_vinfo) = datarefs; /* Analyze the data references. */ @@ -2438,8 +2430,8 @@ vect_slp_analyze_bb_1 (basic_block bb) } /* Mark all the statements that we do not want to vectorize. */ - for (gimple_stmt_iterator gsi = gsi_start_bb (BB_VINFO_BB (bb_vinfo)); - !gsi_end_p (gsi); gsi_next (&gsi)) + for (gimple_stmt_iterator gsi = bb_vinfo->region_begin; + gsi_stmt (gsi) != gsi_stmt (bb_vinfo->region_end); gsi_next (&gsi)) { stmt_vec_info vinfo = vinfo_for_stmt (gsi_stmt (gsi)); if (STMT_SLP_TYPE (vinfo) != pure_slp) @@ -2509,77 +2501,105 @@ bool vect_slp_bb (basic_block bb) { bb_vec_info bb_vinfo; - int insns = 0; gimple_stmt_iterator gsi; unsigned int vector_sizes; + bool any_vectorized = false; if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, "===vect_slp_analyze_bb===\n"); - for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) - { - gimple *stmt = gsi_stmt (gsi); - if (!is_gimple_debug (stmt) - && !gimple_nop_p (stmt) - && gimple_code (stmt) != GIMPLE_LABEL) - insns++; - if (gimple_location (stmt) != UNKNOWN_LOCATION) - vect_location = gimple_location (stmt); - } - - if (insns > PARAM_VALUE (PARAM_SLP_MAX_INSNS_IN_BB)) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "not vectorized: too many instructions in " - "basic block.\n"); - - return false; - } - /* Autodetect first vector size we try. */ current_vector_size = 0; vector_sizes = targetm.vectorize.autovectorize_vector_sizes (); + gsi = gsi_start_bb (bb); + while (1) { - bb_vinfo = vect_slp_analyze_bb_1 (bb); - if (bb_vinfo) - { - if (!dbg_cnt (vect_slp)) - { - destroy_bb_vec_info (bb_vinfo); - return false; - } + if (gsi_end_p (gsi)) + break; + gimple_stmt_iterator region_begin = gsi; + vec datarefs = vNULL; + int insns = 0; + + for (; !gsi_end_p (gsi); gsi_next (&gsi)) + { + gimple *stmt = gsi_stmt (gsi); + if (is_gimple_debug (stmt)) + continue; + insns++; + + if (gimple_location (stmt) != UNKNOWN_LOCATION) + vect_location = gimple_location (stmt); + + if (!find_data_references_in_stmt (NULL, stmt, &datarefs)) + break; + } + + /* Skip leading unhandled stmts. */ + if (gsi_stmt (region_begin) == gsi_stmt (gsi)) + { + gsi_next (&gsi); + continue; + } + + gimple_stmt_iterator region_end = gsi; + + bool vectorized = false; + bb_vinfo = vect_slp_analyze_bb_1 (region_begin, region_end, + datarefs, insns); + if (bb_vinfo + && dbg_cnt (vect_slp)) + { if (dump_enabled_p ()) - dump_printf_loc (MSG_NOTE, vect_location, "SLPing BB\n"); + dump_printf_loc (MSG_NOTE, vect_location, "SLPing BB part\n"); vect_schedule_slp (bb_vinfo); if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, - "BASIC BLOCK VECTORIZED\n"); + "basic block part vectorized\n"); destroy_bb_vec_info (bb_vinfo); - return true; + vectorized = true; } + else + destroy_bb_vec_info (bb_vinfo); - destroy_bb_vec_info (bb_vinfo); + any_vectorized |= vectorized; vector_sizes &= ~current_vector_size; - if (vector_sizes == 0 - || current_vector_size == 0) - return false; + if (vectorized + || vector_sizes == 0 + || current_vector_size == 0) + { + if (gsi_end_p (region_end)) + break; - /* Try the next biggest vector size. */ - current_vector_size = 1 << floor_log2 (vector_sizes); - if (dump_enabled_p ()) - dump_printf_loc (MSG_NOTE, vect_location, - "***** Re-trying analysis with " - "vector size %d\n", current_vector_size); + /* Skip the unhandled stmt. */ + gsi_next (&gsi); + + /* And reset vector sizes. */ + current_vector_size = 0; + vector_sizes = targetm.vectorize.autovectorize_vector_sizes (); + } + else + { + /* Try the next biggest vector size. */ + current_vector_size = 1 << floor_log2 (vector_sizes); + if (dump_enabled_p ()) + dump_printf_loc (MSG_NOTE, vect_location, + "***** Re-trying analysis with " + "vector size %d\n", current_vector_size); + + /* Start over. */ + gsi = region_begin; + } } + + return any_vectorized; } diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index 378973bb8c5..96074fcaa2c 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -8141,12 +8141,7 @@ vect_is_simple_use (tree operand, vec_info *vinfo, dump_gimple_stmt (MSG_NOTE, TDF_SLIM, *def_stmt, 0); } - basic_block bb = gimple_bb (*def_stmt); - if ((is_a (vinfo) - && !flow_bb_inside_loop_p (as_a (vinfo)->loop, bb)) - || (is_a (vinfo) - && (bb != as_a (vinfo)->bb - || gimple_code (*def_stmt) == GIMPLE_PHI))) + if (! vect_stmt_in_region_p (vinfo, *def_stmt)) *dt = vect_external_def; else { diff --git a/gcc/tree-vectorizer.c b/gcc/tree-vectorizer.c index 7b3d9a331a9..41e87a8d9bd 100644 --- a/gcc/tree-vectorizer.c +++ b/gcc/tree-vectorizer.c @@ -350,6 +350,33 @@ vect_destroy_datarefs (vec_info *vinfo) } +/* Return whether STMT is inside the region we try to vectorize. */ + +bool +vect_stmt_in_region_p (vec_info *vinfo, gimple *stmt) +{ + if (!gimple_bb (stmt)) + return false; + + if (loop_vec_info loop_vinfo = dyn_cast (vinfo)) + { + struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); + if (!flow_bb_inside_loop_p (loop, gimple_bb (stmt))) + return false; + } + else + { + bb_vec_info bb_vinfo = as_a (vinfo); + if (gimple_bb (stmt) != BB_VINFO_BB (bb_vinfo) + || gimple_uid (stmt) == -1U + || gimple_code (stmt) == GIMPLE_PHI) + return false; + } + + return true; +} + + /* If LOOP has been versioned during ifcvt, return the internal call guarding it. */ @@ -692,6 +719,14 @@ pass_slp_vectorize::execute (function *fun) scev_initialize (); } + /* Mark all stmts as not belonging to the current region. */ + FOR_EACH_BB_FN (bb, fun) + { + for (gimple_stmt_iterator gsi = gsi_start_bb (bb); !gsi_end_p (gsi); + gsi_next (&gsi)) + gimple_set_uid (gsi_stmt (gsi), -1); + } + init_stmt_vec_info_vec (); FOR_EACH_BB_FN (bb, fun) diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index f77a4eb4313..9cde091bde9 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -390,6 +390,8 @@ nested_in_vect_loop_p (struct loop *loop, gimple *stmt) typedef struct _bb_vec_info : public vec_info { basic_block bb; + gimple_stmt_iterator region_begin; + gimple_stmt_iterator region_end; } *bb_vec_info; #define BB_VINFO_BB(B) (B)->bb @@ -1085,5 +1087,6 @@ void vect_pattern_recog (vec_info *); /* In tree-vectorizer.c. */ unsigned vectorize_loops (void); void vect_destroy_datarefs (vec_info *); +bool vect_stmt_in_region_p (vec_info *, gimple *); #endif /* GCC_TREE_VECTORIZER_H */