From 319e6439f592285e7d931c47285c363cca113788 Mon Sep 17 00:00:00 2001 From: Richard Guenther Date: Fri, 11 May 2012 12:03:10 +0000 Subject: [PATCH] re PR tree-optimization/53295 (Vectorizer support for non-constant strided loads depends on gather support overwriting the data-ref with bogus data) 2012-05-11 Richard Guenther PR tree-optimization/53295 * tree-data-ref.h (stride_of_unit_type_p): Handle non-constant strides. * tree-data-ref.c (dr_analyze_innermost): Allow non-constant strides when analyzing data-references in a loop context. * tree-vect-data-refs.c (vect_mark_for_runtime_alias_test): Reject non-constant strides for now. (vect_enhance_data_refs_alignment): Ignore data references that are strided loads. (vect_analyze_data_ref_access): Handle non-constant strides. (vect_check_strided_load): Verify the data-reference is a load. (vect_analyze_data_refs): Restructure to make strided load support not dependent on gather support. * tree-vect-stmts.c (vectorizable_load): Avoid useless work when doing strided or gather loads. * tree-vect-loop-manip.c (vect_vfa_segment_size): Use integer_zerop to compare stride with zero. From-SVN: r187402 --- gcc/ChangeLog | 20 +++++++++ gcc/tree-data-ref.c | 5 ++- gcc/tree-data-ref.h | 7 +-- gcc/tree-vect-data-refs.c | 89 +++++++++++++++++++++++++++++--------- gcc/tree-vect-loop-manip.c | 2 +- gcc/tree-vect-stmts.c | 67 +++++++++++++++------------- 6 files changed, 132 insertions(+), 58 deletions(-) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 8e8b3fa11f6..c5d733aa88c 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,23 @@ +2012-05-11 Richard Guenther + + PR tree-optimization/53295 + * tree-data-ref.h (stride_of_unit_type_p): Handle non-constant + strides. + * tree-data-ref.c (dr_analyze_innermost): Allow non-constant + strides when analyzing data-references in a loop context. + * tree-vect-data-refs.c (vect_mark_for_runtime_alias_test): Reject + non-constant strides for now. + (vect_enhance_data_refs_alignment): Ignore data references + that are strided loads. + (vect_analyze_data_ref_access): Handle non-constant strides. + (vect_check_strided_load): Verify the data-reference is a load. + (vect_analyze_data_refs): Restructure to make strided load + support not dependent on gather support. + * tree-vect-stmts.c (vectorizable_load): Avoid useless work + when doing strided or gather loads. + * tree-vect-loop-manip.c (vect_vfa_segment_size): Use + integer_zerop to compare stride with zero. + 2012-05-11 H.J. Lu * config/i386/driver-i386.c (host_detect_local_cpu): Support diff --git a/gcc/tree-data-ref.c b/gcc/tree-data-ref.c index 1381b535bd3..90b6f70e4a9 100644 --- a/gcc/tree-data-ref.c +++ b/gcc/tree-data-ref.c @@ -736,7 +736,7 @@ dr_analyze_innermost (struct data_reference *dr, struct loop *nest) if (in_loop) { if (!simple_iv (loop, loop_containing_stmt (stmt), base, &base_iv, - false)) + nest ? true : false)) { if (nest) { @@ -773,7 +773,8 @@ dr_analyze_innermost (struct data_reference *dr, struct loop *nest) offset_iv.step = ssize_int (0); } else if (!simple_iv (loop, loop_containing_stmt (stmt), - poffset, &offset_iv, false)) + poffset, &offset_iv, + nest ? true : false)) { if (nest) { diff --git a/gcc/tree-data-ref.h b/gcc/tree-data-ref.h index 41a20d74206..efce845625f 100644 --- a/gcc/tree-data-ref.h +++ b/gcc/tree-data-ref.h @@ -618,9 +618,10 @@ bool stmt_with_adjacent_zero_store_dr_p (gimple); static inline bool stride_of_unit_type_p (tree stride, tree type) { - return tree_int_cst_equal (fold_unary (ABS_EXPR, TREE_TYPE (stride), - stride), - TYPE_SIZE_UNIT (type)); + return (TREE_CODE (stride) == INTEGER_CST + && tree_int_cst_equal (fold_unary (ABS_EXPR, TREE_TYPE (stride), + stride), + TYPE_SIZE_UNIT (type))); } /* Determines whether RDG vertices V1 and V2 access to similar memory diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c index 715e3ffde61..e536321fcdc 100644 --- a/gcc/tree-vect-data-refs.c +++ b/gcc/tree-vect-data-refs.c @@ -542,6 +542,17 @@ vect_mark_for_runtime_alias_test (ddr_p ddr, loop_vec_info loop_vinfo) return false; } + /* FORNOW: We don't support creating runtime alias tests for non-constant + step. */ + if (TREE_CODE (DR_STEP (DDR_A (ddr))) != INTEGER_CST + || TREE_CODE (DR_STEP (DDR_B (ddr))) != INTEGER_CST) + { + if (vect_print_dump_info (REPORT_DR_DETAILS)) + fprintf (vect_dump, "versioning not yet supported for non-constant " + "step"); + return false; + } + VEC_safe_push (ddr_p, heap, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo), ddr); return true; } @@ -1522,6 +1533,11 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo) if (integer_zerop (DR_STEP (dr))) continue; + /* Strided loads perform only component accesses, alignment is + irrelevant for them. */ + if (STMT_VINFO_STRIDE_LOAD_P (stmt_info)) + continue; + supportable_dr_alignment = vect_supportable_dr_alignment (dr, true); do_peeling = vector_alignment_reachable_p (dr); if (do_peeling) @@ -1779,6 +1795,11 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo) && GROUP_FIRST_ELEMENT (stmt_info) != stmt) continue; + /* Strided loads perform only component accesses, alignment is + irrelevant for them. */ + if (STMT_VINFO_STRIDE_LOAD_P (stmt_info)) + continue; + save_misalignment = DR_MISALIGNMENT (dr); vect_update_misalignment_for_peel (dr, dr0, npeel); supportable_dr_alignment = vect_supportable_dr_alignment (dr, false); @@ -1861,6 +1882,11 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo) && GROUP_FIRST_ELEMENT (stmt_info) != stmt)) continue; + /* Strided loads perform only component accesses, alignment is + irrelevant for them. */ + if (STMT_VINFO_STRIDE_LOAD_P (stmt_info)) + continue; + supportable_dr_alignment = vect_supportable_dr_alignment (dr, false); if (!supportable_dr_alignment) @@ -2329,7 +2355,6 @@ vect_analyze_data_ref_access (struct data_reference *dr) stmt_vec_info stmt_info = vinfo_for_stmt (stmt); loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); struct loop *loop = NULL; - HOST_WIDE_INT dr_step; if (loop_vinfo) loop = LOOP_VINFO_LOOP (loop_vinfo); @@ -2342,8 +2367,7 @@ vect_analyze_data_ref_access (struct data_reference *dr) } /* Allow invariant loads in loops. */ - dr_step = TREE_INT_CST_LOW (step); - if (loop_vinfo && dr_step == 0) + if (loop_vinfo && integer_zerop (step)) { GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)) = NULL; return DR_IS_READ (dr); @@ -2357,9 +2381,7 @@ vect_analyze_data_ref_access (struct data_reference *dr) /* For the rest of the analysis we use the outer-loop step. */ step = STMT_VINFO_DR_STEP (stmt_info); - dr_step = TREE_INT_CST_LOW (step); - - if (dr_step == 0) + if (integer_zerop (step)) { if (vect_print_dump_info (REPORT_ALIGNMENT)) fprintf (vect_dump, "zero step in outer loop."); @@ -2371,13 +2393,17 @@ vect_analyze_data_ref_access (struct data_reference *dr) } /* Consecutive? */ - if (!tree_int_cst_compare (step, TYPE_SIZE_UNIT (scalar_type)) - || (dr_step < 0 - && !compare_tree_int (TYPE_SIZE_UNIT (scalar_type), -dr_step))) + if (TREE_CODE (step) == INTEGER_CST) { - /* Mark that it is not interleaving. */ - GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)) = NULL; - return true; + HOST_WIDE_INT dr_step = TREE_INT_CST_LOW (step); + if (!tree_int_cst_compare (step, TYPE_SIZE_UNIT (scalar_type)) + || (dr_step < 0 + && !compare_tree_int (TYPE_SIZE_UNIT (scalar_type), -dr_step))) + { + /* Mark that it is not interleaving. */ + GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)) = NULL; + return true; + } } if (loop && nested_in_vect_loop_p (loop, stmt)) @@ -2387,6 +2413,10 @@ vect_analyze_data_ref_access (struct data_reference *dr) return false; } + /* Assume this is a DR handled by non-constant strided load case. */ + if (TREE_CODE (step) != INTEGER_CST) + return STMT_VINFO_STRIDE_LOAD_P (stmt_info); + /* Not consecutive access - check if it's a part of interleaving group. */ return vect_analyze_group_access (dr); } @@ -2720,6 +2750,9 @@ vect_check_strided_load (gimple stmt, loop_vec_info loop_vinfo, tree *basep, tree base, off; affine_iv iv; + if (!DR_IS_READ (dr)) + return false; + base = DR_REF (dr); if (TREE_CODE (base) == ARRAY_REF) @@ -3148,21 +3181,19 @@ vect_analyze_data_refs (loop_vec_info loop_vinfo, VEC (ddr_p, heap) *ddrs = LOOP_VINFO_DDRS (loop_vinfo); struct data_dependence_relation *ddr, *newddr; bool bad = false; - bool strided_load = false; tree off; VEC (loop_p, heap) *nest = LOOP_VINFO_LOOP_NEST (loop_vinfo); - strided_load = vect_check_strided_load (stmt, loop_vinfo, NULL, NULL); gather = 0 != vect_check_gather (stmt, loop_vinfo, NULL, &off, NULL); if (gather && get_vectype_for_scalar_type (TREE_TYPE (off)) == NULL_TREE) gather = false; - if (!gather && !strided_load) + if (!gather) { if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS)) { fprintf (vect_dump, - "not vectorized: not suitable for gather/strided load "); + "not vectorized: not suitable for gather load "); print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM); } return false; @@ -3215,16 +3246,32 @@ vect_analyze_data_refs (loop_vec_info loop_vinfo, { fprintf (vect_dump, "not vectorized: data dependence conflict" - " prevents gather/strided load"); + " prevents gather load"); print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM); } return false; } - if (gather) - STMT_VINFO_GATHER_P (stmt_info) = true; - else if (strided_load) - STMT_VINFO_STRIDE_LOAD_P (stmt_info) = true; + STMT_VINFO_GATHER_P (stmt_info) = true; + } + else if (loop_vinfo + && TREE_CODE (DR_STEP (dr)) != INTEGER_CST) + { + bool strided_load = false; + if (!nested_in_vect_loop_p (loop, stmt)) + strided_load + = vect_check_strided_load (stmt, loop_vinfo, NULL, NULL); + if (!strided_load) + { + if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS)) + { + fprintf (vect_dump, + "not vectorized: not suitable for strided load "); + print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM); + } + return false; + } + STMT_VINFO_STRIDE_LOAD_P (stmt_info) = true; } } diff --git a/gcc/tree-vect-loop-manip.c b/gcc/tree-vect-loop-manip.c index 5327e98a240..ac2eb767cf9 100644 --- a/gcc/tree-vect-loop-manip.c +++ b/gcc/tree-vect-loop-manip.c @@ -2334,7 +2334,7 @@ vect_vfa_segment_size (struct data_reference *dr, tree length_factor) { tree segment_length; - if (!compare_tree_int (DR_STEP (dr), 0)) + if (integer_zerop (DR_STEP (dr))) segment_length = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (dr))); else segment_length = size_binop (MULT_EXPR, diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index a0368d83e94..88204fec2d3 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -4210,7 +4210,7 @@ vectorizable_load (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, bool load_lanes_p = false; gimple first_stmt; bool inv_p; - bool negative; + bool negative = false; bool compute_in_loop = false; struct loop *at_loop; int vec_num; @@ -4280,17 +4280,6 @@ vectorizable_load (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, if (!STMT_VINFO_DATA_REF (stmt_info)) return false; - negative = tree_int_cst_compare (nested_in_vect_loop - ? STMT_VINFO_DR_STEP (stmt_info) - : DR_STEP (dr), - size_zero_node) < 0; - if (negative && ncopies > 1) - { - if (vect_print_dump_info (REPORT_DETAILS)) - fprintf (vect_dump, "multiple types with negative step."); - return false; - } - elem_type = TREE_TYPE (vectype); mode = TYPE_MODE (vectype); @@ -4321,24 +4310,6 @@ vectorizable_load (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, } } - if (negative) - { - gcc_assert (!grouped_load && !STMT_VINFO_GATHER_P (stmt_info)); - alignment_support_scheme = vect_supportable_dr_alignment (dr, false); - if (alignment_support_scheme != dr_aligned - && alignment_support_scheme != dr_unaligned_supported) - { - if (vect_print_dump_info (REPORT_DETAILS)) - fprintf (vect_dump, "negative step but alignment required."); - return false; - } - if (!perm_mask_for_reverse (vectype)) - { - if (vect_print_dump_info (REPORT_DETAILS)) - fprintf (vect_dump, "negative step and reversing not supported."); - return false; - } - } if (STMT_VINFO_GATHER_P (stmt_info)) { @@ -4358,7 +4329,41 @@ vectorizable_load (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, } else if (STMT_VINFO_STRIDE_LOAD_P (stmt_info)) { - vect_check_strided_load (stmt, loop_vinfo, &stride_base, &stride_step); + if (!vect_check_strided_load (stmt, loop_vinfo, + &stride_base, &stride_step)) + return false; + } + else + { + negative = tree_int_cst_compare (nested_in_vect_loop + ? STMT_VINFO_DR_STEP (stmt_info) + : DR_STEP (dr), + size_zero_node) < 0; + if (negative && ncopies > 1) + { + if (vect_print_dump_info (REPORT_DETAILS)) + fprintf (vect_dump, "multiple types with negative step."); + return false; + } + + if (negative) + { + gcc_assert (!grouped_load); + alignment_support_scheme = vect_supportable_dr_alignment (dr, false); + if (alignment_support_scheme != dr_aligned + && alignment_support_scheme != dr_unaligned_supported) + { + if (vect_print_dump_info (REPORT_DETAILS)) + fprintf (vect_dump, "negative step but alignment required."); + return false; + } + if (!perm_mask_for_reverse (vectype)) + { + if (vect_print_dump_info (REPORT_DETAILS)) + fprintf (vect_dump, "negative step and reversing not supported."); + return false; + } + } } if (!vec_stmt) /* transformation not required. */