optabs-query.h (get_vcond_mask_icode): New.
gcc/ 2015-11-10 Ilya Enkovich <enkovich.gnu@gmail.com> * optabs-query.h (get_vcond_mask_icode): New. * optabs-tree.c (expand_vec_cond_expr_p): Use get_vcond_mask_icode for VEC_COND_EXPR with mask. * optabs.c (expand_vec_cond_mask_expr): New. (expand_vec_cond_expr): Use get_vcond_mask_icode when possible. * optabs.def (vcond_mask_optab): New. * tree-vect-patterns.c (vect_recog_bool_pattern): Don't generate redundant comparison for COND_EXPR. * tree-vect-stmts.c (vect_is_simple_cond): Allow SSA_NAME as a condition. (vectorizable_condition): Likewise. * tree-vect-slp.c (vect_get_and_check_slp_defs): Allow cond_exp with no embedded comparison. (vect_build_slp_tree_1): Likewise. From-SVN: r230101
This commit is contained in:
parent
1ab8a1b176
commit
a414c77f2a
|
@ -1,3 +1,21 @@
|
|||
2015-11-10 Ilya Enkovich <enkovich.gnu@gmail.com>
|
||||
|
||||
* optabs-query.h (get_vcond_mask_icode): New.
|
||||
* optabs-tree.c (expand_vec_cond_expr_p): Use
|
||||
get_vcond_mask_icode for VEC_COND_EXPR with mask.
|
||||
* optabs.c (expand_vec_cond_mask_expr): New.
|
||||
(expand_vec_cond_expr): Use get_vcond_mask_icode
|
||||
when possible.
|
||||
* optabs.def (vcond_mask_optab): New.
|
||||
* tree-vect-patterns.c (vect_recog_bool_pattern): Don't
|
||||
generate redundant comparison for COND_EXPR.
|
||||
* tree-vect-stmts.c (vect_is_simple_cond): Allow SSA_NAME
|
||||
as a condition.
|
||||
(vectorizable_condition): Likewise.
|
||||
* tree-vect-slp.c (vect_get_and_check_slp_defs): Allow
|
||||
cond_exp with no embedded comparison.
|
||||
(vect_build_slp_tree_1): Likewise.
|
||||
|
||||
2015-11-10 Ilya Enkovich <enkovich.gnu@gmail.com>
|
||||
|
||||
* config/i386/sse.md (maskload<mode>): Rename to ...
|
||||
|
|
|
@ -98,6 +98,15 @@ get_vcond_icode (machine_mode vmode, machine_mode cmode, bool uns)
|
|||
return icode;
|
||||
}
|
||||
|
||||
/* Return insn code for a conditional operator with a mask mode
|
||||
MMODE resulting in a value of mode VMODE. */
|
||||
|
||||
static inline enum insn_code
|
||||
get_vcond_mask_icode (machine_mode vmode, machine_mode mmode)
|
||||
{
|
||||
return convert_optab_handler (vcond_mask_optab, vmode, mmode);
|
||||
}
|
||||
|
||||
/* Enumerates the possible extraction_insn operations. */
|
||||
enum extraction_pattern { EP_insv, EP_extv, EP_extzv };
|
||||
|
||||
|
|
|
@ -342,6 +342,9 @@ expand_vec_cond_expr_p (tree value_type, tree cmp_op_type)
|
|||
{
|
||||
machine_mode value_mode = TYPE_MODE (value_type);
|
||||
machine_mode cmp_op_mode = TYPE_MODE (cmp_op_type);
|
||||
if (VECTOR_BOOLEAN_TYPE_P (cmp_op_type))
|
||||
return get_vcond_mask_icode (TYPE_MODE (value_type),
|
||||
TYPE_MODE (cmp_op_type)) != CODE_FOR_nothing;
|
||||
if (GET_MODE_SIZE (value_mode) != GET_MODE_SIZE (cmp_op_mode)
|
||||
|| GET_MODE_NUNITS (value_mode) != GET_MODE_NUNITS (cmp_op_mode)
|
||||
|| get_vcond_icode (TYPE_MODE (value_type), TYPE_MODE (cmp_op_type),
|
||||
|
|
49
gcc/optabs.c
49
gcc/optabs.c
|
@ -5426,6 +5426,38 @@ expand_vec_perm (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target)
|
|||
return tmp;
|
||||
}
|
||||
|
||||
/* Generate insns for a VEC_COND_EXPR with mask, given its TYPE and its
|
||||
three operands. */
|
||||
|
||||
rtx
|
||||
expand_vec_cond_mask_expr (tree vec_cond_type, tree op0, tree op1, tree op2,
|
||||
rtx target)
|
||||
{
|
||||
struct expand_operand ops[4];
|
||||
machine_mode mode = TYPE_MODE (vec_cond_type);
|
||||
machine_mode mask_mode = TYPE_MODE (TREE_TYPE (op0));
|
||||
enum insn_code icode = get_vcond_mask_icode (mode, mask_mode);
|
||||
rtx mask, rtx_op1, rtx_op2;
|
||||
|
||||
if (icode == CODE_FOR_nothing)
|
||||
return 0;
|
||||
|
||||
mask = expand_normal (op0);
|
||||
rtx_op1 = expand_normal (op1);
|
||||
rtx_op2 = expand_normal (op2);
|
||||
|
||||
mask = force_reg (GET_MODE (mask), mask);
|
||||
rtx_op1 = force_reg (GET_MODE (rtx_op1), rtx_op1);
|
||||
|
||||
create_output_operand (&ops[0], target, mode);
|
||||
create_input_operand (&ops[1], rtx_op1, mode);
|
||||
create_input_operand (&ops[2], rtx_op2, mode);
|
||||
create_input_operand (&ops[3], mask, mask_mode);
|
||||
expand_insn (icode, 4, ops);
|
||||
|
||||
return ops[0].value;
|
||||
}
|
||||
|
||||
/* Generate insns for a VEC_COND_EXPR, given its TYPE and its
|
||||
three operands. */
|
||||
|
||||
|
@ -5450,11 +5482,20 @@ expand_vec_cond_expr (tree vec_cond_type, tree op0, tree op1, tree op2,
|
|||
}
|
||||
else
|
||||
{
|
||||
/* Fake op0 < 0. */
|
||||
gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0)));
|
||||
op0a = op0;
|
||||
op0b = build_zero_cst (TREE_TYPE (op0));
|
||||
tcode = LT_EXPR;
|
||||
if (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0)))
|
||||
!= CODE_FOR_nothing)
|
||||
return expand_vec_cond_mask_expr (vec_cond_type, op0, op1,
|
||||
op2, target);
|
||||
/* Fake op0 < 0. */
|
||||
else
|
||||
{
|
||||
gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0)))
|
||||
== MODE_VECTOR_INT);
|
||||
op0a = op0;
|
||||
op0b = build_zero_cst (TREE_TYPE (op0));
|
||||
tcode = LT_EXPR;
|
||||
}
|
||||
}
|
||||
cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a));
|
||||
unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
|
||||
|
|
|
@ -61,6 +61,7 @@ OPTAB_CD(vec_load_lanes_optab, "vec_load_lanes$a$b")
|
|||
OPTAB_CD(vec_store_lanes_optab, "vec_store_lanes$a$b")
|
||||
OPTAB_CD(vcond_optab, "vcond$a$b")
|
||||
OPTAB_CD(vcondu_optab, "vcondu$a$b")
|
||||
OPTAB_CD(vcond_mask_optab, "vcond_mask_$a$b")
|
||||
OPTAB_CD(vec_cmp_optab, "vec_cmp$a$b")
|
||||
OPTAB_CD(vec_cmpu_optab, "vec_cmpu$a$b")
|
||||
OPTAB_CD(maskload_optab, "maskload$a$b")
|
||||
|
|
|
@ -3302,7 +3302,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in,
|
|||
else
|
||||
{
|
||||
tree type = search_type_for_mask (var, vinfo);
|
||||
tree cst0, cst1, cmp, tmp;
|
||||
tree cst0, cst1, tmp;
|
||||
|
||||
if (!type)
|
||||
return NULL;
|
||||
|
@ -3318,9 +3318,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in,
|
|||
cst0 = build_int_cst (type, 0);
|
||||
cst1 = build_int_cst (type, 1);
|
||||
tmp = vect_recog_temp_ssa_var (type, NULL);
|
||||
cmp = build2 (NE_EXPR, boolean_type_node,
|
||||
var, build_int_cst (TREE_TYPE (var), 0));
|
||||
pattern_stmt = gimple_build_assign (tmp, COND_EXPR, cmp, cst1, cst0);
|
||||
pattern_stmt = gimple_build_assign (tmp, COND_EXPR, var, cst1, cst0);
|
||||
|
||||
if (!useless_type_conversion_p (type, TREE_TYPE (lhs)))
|
||||
{
|
||||
|
@ -3363,19 +3361,16 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in,
|
|||
if (get_vectype_for_scalar_type (type) == NULL_TREE)
|
||||
return NULL;
|
||||
|
||||
if (check_bool_pattern (var, vinfo))
|
||||
{
|
||||
rhs = adjust_bool_pattern (var, type, NULL_TREE, stmts);
|
||||
rhs = build2 (NE_EXPR, boolean_type_node,
|
||||
rhs, build_int_cst (type, 0));
|
||||
}
|
||||
else
|
||||
rhs = build2 (NE_EXPR, boolean_type_node,
|
||||
var, build_int_cst (TREE_TYPE (var), 0)),
|
||||
if (!check_bool_pattern (var, vinfo))
|
||||
return NULL;
|
||||
|
||||
rhs = adjust_bool_pattern (var, type, NULL_TREE, stmts);
|
||||
|
||||
lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
|
||||
pattern_stmt
|
||||
= gimple_build_assign (lhs, COND_EXPR, rhs,
|
||||
= gimple_build_assign (lhs, COND_EXPR,
|
||||
build2 (NE_EXPR, boolean_type_node,
|
||||
rhs, build_int_cst (type, 0)),
|
||||
gimple_assign_rhs2 (last_stmt),
|
||||
gimple_assign_rhs3 (last_stmt));
|
||||
*type_out = vectype;
|
||||
|
@ -3402,7 +3397,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in,
|
|||
else
|
||||
{
|
||||
tree type = search_type_for_mask (var, vinfo);
|
||||
tree cst0, cst1, cmp, new_vectype;
|
||||
tree cst0, cst1, new_vectype;
|
||||
|
||||
if (!type)
|
||||
return NULL;
|
||||
|
@ -3415,10 +3410,7 @@ vect_recog_bool_pattern (vec<gimple *> *stmts, tree *type_in,
|
|||
new_vectype = get_vectype_for_scalar_type (type);
|
||||
|
||||
rhs = vect_recog_temp_ssa_var (type, NULL);
|
||||
cmp = build2 (NE_EXPR, boolean_type_node,
|
||||
var, build_int_cst (TREE_TYPE (var), 0));
|
||||
pattern_stmt = gimple_build_assign (rhs, COND_EXPR,
|
||||
cmp, cst1, cst0);
|
||||
pattern_stmt = gimple_build_assign (rhs, COND_EXPR, var, cst1, cst0);
|
||||
|
||||
pattern_stmt_info = new_stmt_vec_info (pattern_stmt, vinfo);
|
||||
set_vinfo_for_stmt (pattern_stmt, pattern_stmt_info);
|
||||
|
|
|
@ -226,7 +226,8 @@ vect_get_and_check_slp_defs (vec_info *vinfo,
|
|||
{
|
||||
enum tree_code code = gimple_assign_rhs_code (stmt);
|
||||
number_of_oprnds = gimple_num_ops (stmt) - 1;
|
||||
if (gimple_assign_rhs_code (stmt) == COND_EXPR)
|
||||
if (gimple_assign_rhs_code (stmt) == COND_EXPR
|
||||
&& COMPARISON_CLASS_P (gimple_assign_rhs1 (stmt)))
|
||||
{
|
||||
first_op_cond = true;
|
||||
commutative = true;
|
||||
|
@ -447,7 +448,6 @@ vect_build_slp_tree_1 (vec_info *vinfo,
|
|||
machine_mode vec_mode;
|
||||
HOST_WIDE_INT dummy;
|
||||
gimple *first_load = NULL, *prev_first_load = NULL;
|
||||
tree cond;
|
||||
|
||||
/* For every stmt in NODE find its def stmt/s. */
|
||||
FOR_EACH_VEC_ELT (stmts, i, stmt)
|
||||
|
@ -492,24 +492,6 @@ vect_build_slp_tree_1 (vec_info *vinfo,
|
|||
return false;
|
||||
}
|
||||
|
||||
if (is_gimple_assign (stmt)
|
||||
&& gimple_assign_rhs_code (stmt) == COND_EXPR
|
||||
&& (cond = gimple_assign_rhs1 (stmt))
|
||||
&& !COMPARISON_CLASS_P (cond))
|
||||
{
|
||||
if (dump_enabled_p ())
|
||||
{
|
||||
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
|
||||
"Build SLP failed: condition is not "
|
||||
"comparison ");
|
||||
dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM, stmt, 0);
|
||||
dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
|
||||
}
|
||||
/* Fatal mismatch. */
|
||||
matches[0] = false;
|
||||
return false;
|
||||
}
|
||||
|
||||
scalar_type = vect_get_smallest_scalar_type (stmt, &dummy, &dummy);
|
||||
vectype = get_vectype_for_scalar_type (scalar_type);
|
||||
if (!vectype)
|
||||
|
|
|
@ -7168,6 +7168,19 @@ vect_is_simple_cond (tree cond, vec_info *vinfo, tree *comp_vectype)
|
|||
enum vect_def_type dt;
|
||||
tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
|
||||
|
||||
/* Mask case. */
|
||||
if (TREE_CODE (cond) == SSA_NAME
|
||||
&& TREE_CODE (TREE_TYPE (cond)) == BOOLEAN_TYPE)
|
||||
{
|
||||
gimple *lhs_def_stmt = SSA_NAME_DEF_STMT (cond);
|
||||
if (!vect_is_simple_use (cond, vinfo, &lhs_def_stmt,
|
||||
&dt, comp_vectype)
|
||||
|| !*comp_vectype
|
||||
|| !VECTOR_BOOLEAN_TYPE_P (*comp_vectype))
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!COMPARISON_CLASS_P (cond))
|
||||
return false;
|
||||
|
||||
|
@ -7237,6 +7250,7 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi,
|
|||
vec<tree> vec_oprnds2 = vNULL;
|
||||
vec<tree> vec_oprnds3 = vNULL;
|
||||
tree vec_cmp_type;
|
||||
bool masked = false;
|
||||
|
||||
if (reduc_index && STMT_SLP_TYPE (stmt_info))
|
||||
return false;
|
||||
|
@ -7296,7 +7310,13 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi,
|
|||
if (!vect_is_simple_use (else_clause, stmt_info->vinfo, &def_stmt, &dt))
|
||||
return false;
|
||||
|
||||
vec_cmp_type = build_same_sized_truth_vector_type (comp_vectype);
|
||||
if (VECTOR_BOOLEAN_TYPE_P (comp_vectype))
|
||||
{
|
||||
vec_cmp_type = comp_vectype;
|
||||
masked = true;
|
||||
}
|
||||
else
|
||||
vec_cmp_type = build_same_sized_truth_vector_type (comp_vectype);
|
||||
if (vec_cmp_type == NULL_TREE)
|
||||
return false;
|
||||
|
||||
|
@ -7331,14 +7351,20 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi,
|
|||
auto_vec<tree, 4> ops;
|
||||
auto_vec<vec<tree>, 4> vec_defs;
|
||||
|
||||
ops.safe_push (TREE_OPERAND (cond_expr, 0));
|
||||
ops.safe_push (TREE_OPERAND (cond_expr, 1));
|
||||
if (masked)
|
||||
ops.safe_push (cond_expr);
|
||||
else
|
||||
{
|
||||
ops.safe_push (TREE_OPERAND (cond_expr, 0));
|
||||
ops.safe_push (TREE_OPERAND (cond_expr, 1));
|
||||
}
|
||||
ops.safe_push (then_clause);
|
||||
ops.safe_push (else_clause);
|
||||
vect_get_slp_defs (ops, slp_node, &vec_defs, -1);
|
||||
vec_oprnds3 = vec_defs.pop ();
|
||||
vec_oprnds2 = vec_defs.pop ();
|
||||
vec_oprnds1 = vec_defs.pop ();
|
||||
if (!masked)
|
||||
vec_oprnds1 = vec_defs.pop ();
|
||||
vec_oprnds0 = vec_defs.pop ();
|
||||
|
||||
ops.release ();
|
||||
|
@ -7347,17 +7373,28 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi,
|
|||
else
|
||||
{
|
||||
gimple *gtemp;
|
||||
vec_cond_lhs =
|
||||
vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0),
|
||||
stmt, comp_vectype);
|
||||
vect_is_simple_use (TREE_OPERAND (cond_expr, 0),
|
||||
loop_vinfo, >emp, &dts[0]);
|
||||
if (masked)
|
||||
{
|
||||
vec_cond_lhs
|
||||
= vect_get_vec_def_for_operand (cond_expr, stmt,
|
||||
comp_vectype);
|
||||
vect_is_simple_use (cond_expr, stmt_info->vinfo,
|
||||
>emp, &dts[0]);
|
||||
}
|
||||
else
|
||||
{
|
||||
vec_cond_lhs =
|
||||
vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0),
|
||||
stmt, comp_vectype);
|
||||
vect_is_simple_use (TREE_OPERAND (cond_expr, 0),
|
||||
loop_vinfo, >emp, &dts[0]);
|
||||
|
||||
vec_cond_rhs =
|
||||
vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1),
|
||||
stmt, comp_vectype);
|
||||
vect_is_simple_use (TREE_OPERAND (cond_expr, 1),
|
||||
loop_vinfo, >emp, &dts[1]);
|
||||
vec_cond_rhs =
|
||||
vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1),
|
||||
stmt, comp_vectype);
|
||||
vect_is_simple_use (TREE_OPERAND (cond_expr, 1),
|
||||
loop_vinfo, >emp, &dts[1]);
|
||||
}
|
||||
if (reduc_index == 1)
|
||||
vec_then_clause = reduc_def;
|
||||
else
|
||||
|
@ -7379,10 +7416,14 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi,
|
|||
}
|
||||
else
|
||||
{
|
||||
vec_cond_lhs = vect_get_vec_def_for_stmt_copy (dts[0],
|
||||
vec_oprnds0.pop ());
|
||||
vec_cond_rhs = vect_get_vec_def_for_stmt_copy (dts[1],
|
||||
vec_oprnds1.pop ());
|
||||
vec_cond_lhs
|
||||
= vect_get_vec_def_for_stmt_copy (dts[0],
|
||||
vec_oprnds0.pop ());
|
||||
if (!masked)
|
||||
vec_cond_rhs
|
||||
= vect_get_vec_def_for_stmt_copy (dts[1],
|
||||
vec_oprnds1.pop ());
|
||||
|
||||
vec_then_clause = vect_get_vec_def_for_stmt_copy (dts[2],
|
||||
vec_oprnds2.pop ());
|
||||
vec_else_clause = vect_get_vec_def_for_stmt_copy (dts[3],
|
||||
|
@ -7392,7 +7433,8 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi,
|
|||
if (!slp_node)
|
||||
{
|
||||
vec_oprnds0.quick_push (vec_cond_lhs);
|
||||
vec_oprnds1.quick_push (vec_cond_rhs);
|
||||
if (!masked)
|
||||
vec_oprnds1.quick_push (vec_cond_rhs);
|
||||
vec_oprnds2.quick_push (vec_then_clause);
|
||||
vec_oprnds3.quick_push (vec_else_clause);
|
||||
}
|
||||
|
@ -7400,12 +7442,17 @@ vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi,
|
|||
/* Arguments are ready. Create the new vector stmt. */
|
||||
FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_cond_lhs)
|
||||
{
|
||||
vec_cond_rhs = vec_oprnds1[i];
|
||||
vec_then_clause = vec_oprnds2[i];
|
||||
vec_else_clause = vec_oprnds3[i];
|
||||
|
||||
vec_compare = build2 (TREE_CODE (cond_expr), vec_cmp_type,
|
||||
vec_cond_lhs, vec_cond_rhs);
|
||||
if (masked)
|
||||
vec_compare = vec_cond_lhs;
|
||||
else
|
||||
{
|
||||
vec_cond_rhs = vec_oprnds1[i];
|
||||
vec_compare = build2 (TREE_CODE (cond_expr), vec_cmp_type,
|
||||
vec_cond_lhs, vec_cond_rhs);
|
||||
}
|
||||
vec_cond_expr = build3 (VEC_COND_EXPR, vectype,
|
||||
vec_compare, vec_then_clause, vec_else_clause);
|
||||
|
||||
|
|
Loading…
Reference in New Issue