remove STMT_VINFO_NUM_SLP_USES
This removes STMT_VINFO_NUM_SLP_USES by pushing the setting of the shared stmt_vec_info vector type to where we actually need it which is alignment analysis and vectorizable_* analysis (where we could eventually elide it for non-load/store operations). In particular "uses" in the cache and in disqualified SLP subgraphs should no longer provide conflicting vector types this way. 2020-09-16 Richard Biener <rguenther@suse.de> * tree-vectorizer.h (_stmt_vec_info::num_slp_uses): Remove. (STMT_VINFO_NUM_SLP_USES): Likewise. (vect_free_slp_instance): Adjust. (vect_update_shared_vectype): Declare. * tree-vectorizer.c (vec_info::~vec_info): Adjust. * tree-vect-loop.c (vect_analyze_loop_2): Likewise. (vectorizable_live_operation): Use vector type from SLP_TREE_REPRESENTATIVE. (vect_transform_loop): Adjust. * tree-vect-data-refs.c (vect_slp_analyze_node_alignment): Set the shared vector type. * tree-vect-slp.c (vect_free_slp_tree): Remove final_p parameter, remove STMT_VINFO_NUM_SLP_USES updating. (vect_free_slp_instance): Adjust. (vect_create_new_slp_node): Remove STMT_VINFO_NUM_SLP_USES updating. (vect_update_shared_vectype): Always compare with the present vector type, update if NULL. (vect_build_slp_tree_1): Do not update the shared vector type here. (vect_build_slp_tree_2): Adjust. (slp_copy_subtree): Likewise. (vect_attempt_slp_rearrange_stmts): Likewise. (vect_analyze_slp_instance): Likewise. (vect_analyze_slp): Likewise. (vect_slp_analyze_node_operations_1): Update the shared vector type. (vect_slp_analyze_operations): Adjust. (vect_slp_analyze_bb_1): Likewise.
This commit is contained in:
Richard Biener
parent
8a4f343aa3
commit
f02b2077c0
|
|
@ -2380,6 +2380,11 @@ vect_slp_analyze_node_alignment (vec_info *vinfo, slp_tree node)
|
|||
if (SLP_TREE_LOAD_PERMUTATION (node).exists ())
|
||||
first_stmt_info = DR_GROUP_FIRST_ELEMENT (first_stmt_info);
|
||||
|
||||
/* We need to commit to a vector type for the group now. */
|
||||
if (is_a <bb_vec_info> (vinfo)
|
||||
&& !vect_update_shared_vectype (first_stmt_info, SLP_TREE_VECTYPE (node)))
|
||||
return false;
|
||||
|
||||
dr_vec_info *dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
|
||||
vect_compute_data_ref_alignment (vinfo, dr_info);
|
||||
/* In several places we need alignment of the first element anyway. */
|
||||
|
|
|
|||
|
|
@ -2467,7 +2467,7 @@ again:
|
|||
LOOP_VINFO_VECT_FACTOR (loop_vinfo) = saved_vectorization_factor;
|
||||
/* Free the SLP instances. */
|
||||
FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (loop_vinfo), j, instance)
|
||||
vect_free_slp_instance (instance, false);
|
||||
vect_free_slp_instance (instance);
|
||||
LOOP_VINFO_SLP_INSTANCES (loop_vinfo).release ();
|
||||
/* Reset SLP type to loop_vect on all stmts. */
|
||||
for (i = 0; i < LOOP_VINFO_LOOP (loop_vinfo)->num_nodes; ++i)
|
||||
|
|
@ -8031,7 +8031,9 @@ vectorizable_live_operation (vec_info *vinfo,
|
|||
loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
|
||||
imm_use_iterator imm_iter;
|
||||
tree lhs, lhs_type, bitsize, vec_bitsize;
|
||||
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
|
||||
tree vectype = (slp_node
|
||||
? SLP_TREE_VECTYPE (SLP_TREE_REPRESENTATIVE (slp_node))
|
||||
: STMT_VINFO_VECTYPE (stmt_info));
|
||||
poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
|
||||
int ncopies;
|
||||
gimple *use_stmt;
|
||||
|
|
@ -9260,7 +9262,7 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
|
|||
won't work. */
|
||||
slp_instance instance;
|
||||
FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (loop_vinfo), i, instance)
|
||||
vect_free_slp_instance (instance, true);
|
||||
vect_free_slp_instance (instance);
|
||||
LOOP_VINFO_SLP_INSTANCES (loop_vinfo).release ();
|
||||
/* Clear-up safelen field since its value is invalid after vectorization
|
||||
since vectorized loop can have loop-carried dependencies. */
|
||||
|
|
|
|||
|
|
@ -84,12 +84,10 @@ _slp_tree::~_slp_tree ()
|
|||
SLP_TREE_LANE_PERMUTATION (this).release ();
|
||||
}
|
||||
|
||||
/* Recursively free the memory allocated for the SLP tree rooted at NODE.
|
||||
FINAL_P is true if we have vectorized the instance or if we have
|
||||
made a final decision not to vectorize the statements in any way. */
|
||||
/* Recursively free the memory allocated for the SLP tree rooted at NODE. */
|
||||
|
||||
static void
|
||||
vect_free_slp_tree (slp_tree node, bool final_p)
|
||||
vect_free_slp_tree (slp_tree node)
|
||||
{
|
||||
int i;
|
||||
slp_tree child;
|
||||
|
|
@ -98,21 +96,7 @@ vect_free_slp_tree (slp_tree node, bool final_p)
|
|||
return;
|
||||
|
||||
FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
|
||||
vect_free_slp_tree (child, final_p);
|
||||
|
||||
/* Don't update STMT_VINFO_NUM_SLP_USES if it isn't relevant.
|
||||
Some statements might no longer exist, after having been
|
||||
removed by vect_transform_stmt. Updating the remaining
|
||||
statements would be redundant. */
|
||||
if (!final_p)
|
||||
{
|
||||
stmt_vec_info stmt_info;
|
||||
FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
|
||||
{
|
||||
gcc_assert (STMT_VINFO_NUM_SLP_USES (stmt_info) > 0);
|
||||
STMT_VINFO_NUM_SLP_USES (stmt_info)--;
|
||||
}
|
||||
}
|
||||
vect_free_slp_tree (child);
|
||||
|
||||
delete node;
|
||||
}
|
||||
|
|
@ -129,14 +113,12 @@ _slp_instance::location () const
|
|||
}
|
||||
|
||||
|
||||
/* Free the memory allocated for the SLP instance. FINAL_P is true if we
|
||||
have vectorized the instance or if we have made a final decision not
|
||||
to vectorize the statements in any way. */
|
||||
/* Free the memory allocated for the SLP instance. */
|
||||
|
||||
void
|
||||
vect_free_slp_instance (slp_instance instance, bool final_p)
|
||||
vect_free_slp_instance (slp_instance instance)
|
||||
{
|
||||
vect_free_slp_tree (SLP_INSTANCE_TREE (instance), final_p);
|
||||
vect_free_slp_tree (SLP_INSTANCE_TREE (instance));
|
||||
SLP_INSTANCE_LOADS (instance).release ();
|
||||
instance->subgraph_entries.release ();
|
||||
instance->cost_vec.release ();
|
||||
|
|
@ -155,12 +137,6 @@ vect_create_new_slp_node (vec<stmt_vec_info> scalar_stmts, unsigned nops)
|
|||
SLP_TREE_DEF_TYPE (node) = vect_internal_def;
|
||||
SLP_TREE_REPRESENTATIVE (node) = scalar_stmts[0];
|
||||
SLP_TREE_LANES (node) = scalar_stmts.length ();
|
||||
|
||||
unsigned i;
|
||||
stmt_vec_info stmt_info;
|
||||
FOR_EACH_VEC_ELT (scalar_stmts, i, stmt_info)
|
||||
STMT_VINFO_NUM_SLP_USES (stmt_info)++;
|
||||
|
||||
return node;
|
||||
}
|
||||
|
||||
|
|
@ -614,23 +590,23 @@ again:
|
|||
Return true if we can, meaning that this choice doesn't conflict with
|
||||
existing SLP nodes that use STMT_INFO. */
|
||||
|
||||
static bool
|
||||
bool
|
||||
vect_update_shared_vectype (stmt_vec_info stmt_info, tree vectype)
|
||||
{
|
||||
tree old_vectype = STMT_VINFO_VECTYPE (stmt_info);
|
||||
if (old_vectype && useless_type_conversion_p (vectype, old_vectype))
|
||||
return true;
|
||||
if (old_vectype)
|
||||
return useless_type_conversion_p (vectype, old_vectype);
|
||||
|
||||
if (STMT_VINFO_GROUPED_ACCESS (stmt_info)
|
||||
&& DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)))
|
||||
if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
|
||||
{
|
||||
/* We maintain the invariant that if any statement in the group is
|
||||
used, all other members of the group have the same vector type. */
|
||||
stmt_vec_info first_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
|
||||
stmt_vec_info member_info = first_info;
|
||||
for (; member_info; member_info = DR_GROUP_NEXT_ELEMENT (member_info))
|
||||
if (STMT_VINFO_NUM_SLP_USES (member_info) > 0
|
||||
|| is_pattern_stmt_p (member_info))
|
||||
if (is_pattern_stmt_p (member_info)
|
||||
&& !useless_type_conversion_p (vectype,
|
||||
STMT_VINFO_VECTYPE (member_info)))
|
||||
break;
|
||||
|
||||
if (!member_info)
|
||||
|
|
@ -641,8 +617,7 @@ vect_update_shared_vectype (stmt_vec_info stmt_info, tree vectype)
|
|||
return true;
|
||||
}
|
||||
}
|
||||
else if (STMT_VINFO_NUM_SLP_USES (stmt_info) == 0
|
||||
&& !is_pattern_stmt_p (stmt_info))
|
||||
else if (!is_pattern_stmt_p (stmt_info))
|
||||
{
|
||||
STMT_VINFO_VECTYPE (stmt_info) = vectype;
|
||||
return true;
|
||||
|
|
@ -824,10 +799,6 @@ vect_build_slp_tree_1 (vec_info *vinfo, unsigned char *swap,
|
|||
|
||||
gcc_assert (vectype);
|
||||
|
||||
if (is_a <bb_vec_info> (vinfo)
|
||||
&& !vect_update_shared_vectype (stmt_info, vectype))
|
||||
continue;
|
||||
|
||||
gcall *call_stmt = dyn_cast <gcall *> (stmt);
|
||||
if (call_stmt)
|
||||
{
|
||||
|
|
@ -1561,7 +1532,7 @@ fail:
|
|||
|
||||
gcc_assert (child == NULL);
|
||||
FOR_EACH_VEC_ELT (children, j, child)
|
||||
vect_free_slp_tree (child, false);
|
||||
vect_free_slp_tree (child);
|
||||
vect_free_oprnd_info (oprnds_info);
|
||||
return NULL;
|
||||
}
|
||||
|
|
@ -1589,7 +1560,7 @@ fail:
|
|||
/* Roll back. */
|
||||
matches[0] = false;
|
||||
FOR_EACH_VEC_ELT (children, j, child)
|
||||
vect_free_slp_tree (child, false);
|
||||
vect_free_slp_tree (child);
|
||||
|
||||
if (dump_enabled_p ())
|
||||
dump_printf_loc (MSG_NOTE, vect_location,
|
||||
|
|
@ -1833,12 +1804,7 @@ slp_copy_subtree (slp_tree node, hash_map<slp_tree, slp_tree> &map)
|
|||
copy->max_nunits = node->max_nunits;
|
||||
copy->refcnt = 0;
|
||||
if (SLP_TREE_SCALAR_STMTS (node).exists ())
|
||||
{
|
||||
SLP_TREE_SCALAR_STMTS (copy) = SLP_TREE_SCALAR_STMTS (node).copy ();
|
||||
stmt_vec_info stmt_info;
|
||||
FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
|
||||
STMT_VINFO_NUM_SLP_USES (stmt_info)++;
|
||||
}
|
||||
SLP_TREE_SCALAR_STMTS (copy) = SLP_TREE_SCALAR_STMTS (node).copy ();
|
||||
if (SLP_TREE_SCALAR_OPS (node).exists ())
|
||||
SLP_TREE_SCALAR_OPS (copy) = SLP_TREE_SCALAR_OPS (node).copy ();
|
||||
if (SLP_TREE_LOAD_PERMUTATION (node).exists ())
|
||||
|
|
@ -1968,7 +1934,7 @@ vect_attempt_slp_rearrange_stmts (slp_instance slp_instn)
|
|||
/* We have to unshare the SLP tree we modify. */
|
||||
hash_map<slp_tree, slp_tree> map;
|
||||
slp_tree unshared = slp_copy_subtree (SLP_INSTANCE_TREE (slp_instn), map);
|
||||
vect_free_slp_tree (SLP_INSTANCE_TREE (slp_instn), false);
|
||||
vect_free_slp_tree (SLP_INSTANCE_TREE (slp_instn));
|
||||
unshared->refcnt++;
|
||||
SLP_INSTANCE_TREE (slp_instn) = unshared;
|
||||
FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (slp_instn), i, node)
|
||||
|
|
@ -2280,7 +2246,7 @@ vect_analyze_slp_instance (vec_info *vinfo,
|
|||
"Build SLP failed: store group "
|
||||
"size not a multiple of the vector size "
|
||||
"in basic block SLP\n");
|
||||
vect_free_slp_tree (node, false);
|
||||
vect_free_slp_tree (node);
|
||||
return false;
|
||||
}
|
||||
/* Fatal mismatch. */
|
||||
|
|
@ -2290,7 +2256,7 @@ vect_analyze_slp_instance (vec_info *vinfo,
|
|||
"splitting\n");
|
||||
matches[0] = true;
|
||||
matches[group_size / const_max_nunits * const_max_nunits] = false;
|
||||
vect_free_slp_tree (node, false);
|
||||
vect_free_slp_tree (node);
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
@ -2351,7 +2317,7 @@ vect_analyze_slp_instance (vec_info *vinfo,
|
|||
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
|
||||
"Built SLP cancelled: can use "
|
||||
"load/store-lanes\n");
|
||||
vect_free_slp_instance (new_instance, false);
|
||||
vect_free_slp_instance (new_instance);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
@ -2517,7 +2483,7 @@ vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size)
|
|||
for (scalar_stmts_to_slp_tree_map_t::iterator it = bst_map->begin ();
|
||||
it != bst_map->end (); ++it)
|
||||
if ((*it).second)
|
||||
vect_free_slp_tree ((*it).second, false);
|
||||
vect_free_slp_tree ((*it).second);
|
||||
delete bst_map;
|
||||
|
||||
/* Optimize permutations in SLP reductions. */
|
||||
|
|
@ -2839,6 +2805,10 @@ vect_slp_analyze_node_operations_1 (vec_info *vinfo, slp_tree node,
|
|||
if (SLP_TREE_CODE (node) == VEC_PERM_EXPR)
|
||||
return vectorizable_slp_permutation (vinfo, NULL, node, cost_vec);
|
||||
|
||||
if (is_a <bb_vec_info> (vinfo)
|
||||
&& !vect_update_shared_vectype (stmt_info, SLP_TREE_VECTYPE (node)))
|
||||
return false;
|
||||
|
||||
bool dummy;
|
||||
return vect_analyze_stmt (vinfo, stmt_info, &dummy,
|
||||
node, node_instance, cost_vec);
|
||||
|
|
@ -3164,7 +3134,7 @@ vect_slp_analyze_operations (vec_info *vinfo)
|
|||
dump_printf_loc (MSG_NOTE, vect_location,
|
||||
"removing SLP instance operations starting from: %G",
|
||||
stmt_info->stmt);
|
||||
vect_free_slp_instance (instance, false);
|
||||
vect_free_slp_instance (instance);
|
||||
vinfo->slp_instances.ordered_remove (i);
|
||||
cost_vec.release ();
|
||||
}
|
||||
|
|
@ -3590,7 +3560,7 @@ vect_slp_analyze_bb_1 (bb_vec_info bb_vinfo, int n_stmts, bool &fatal,
|
|||
dump_printf_loc (MSG_NOTE, vect_location,
|
||||
"removing SLP instance operations starting from: %G",
|
||||
stmt_info->stmt);
|
||||
vect_free_slp_instance (instance, false);
|
||||
vect_free_slp_instance (instance);
|
||||
BB_VINFO_SLP_INSTANCES (bb_vinfo).ordered_remove (i);
|
||||
continue;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -473,7 +473,7 @@ vec_info::~vec_info ()
|
|||
unsigned int i;
|
||||
|
||||
FOR_EACH_VEC_ELT (slp_instances, i, instance)
|
||||
vect_free_slp_instance (instance, true);
|
||||
vect_free_slp_instance (instance);
|
||||
|
||||
destroy_cost_data (target_cost_data);
|
||||
free_stmt_vec_infos ();
|
||||
|
|
|
|||
|
|
@ -1223,9 +1223,6 @@ public:
|
|||
/* Whether on this stmt reduction meta is recorded. */
|
||||
bool is_reduc_info;
|
||||
|
||||
/* The number of scalar stmt references from active SLP instances. */
|
||||
unsigned int num_slp_uses;
|
||||
|
||||
/* If nonzero, the lhs of the statement could be truncated to this
|
||||
many bits without affecting any users of the result. */
|
||||
unsigned int min_output_precision;
|
||||
|
|
@ -1340,7 +1337,6 @@ struct gather_scatter_info {
|
|||
#define STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED(S) (S)->loop_phi_evolution_base_unchanged
|
||||
#define STMT_VINFO_LOOP_PHI_EVOLUTION_PART(S) (S)->loop_phi_evolution_part
|
||||
#define STMT_VINFO_MIN_NEG_DIST(S) (S)->min_neg_dist
|
||||
#define STMT_VINFO_NUM_SLP_USES(S) (S)->num_slp_uses
|
||||
#define STMT_VINFO_REDUC_TYPE(S) (S)->reduc_type
|
||||
#define STMT_VINFO_REDUC_CODE(S) (S)->reduc_code
|
||||
#define STMT_VINFO_REDUC_FN(S) (S)->reduc_fn
|
||||
|
|
@ -2024,7 +2020,7 @@ extern int vect_get_known_peeling_cost (loop_vec_info, int, int *,
|
|||
extern tree cse_and_gimplify_to_preheader (loop_vec_info, tree);
|
||||
|
||||
/* In tree-vect-slp.c. */
|
||||
extern void vect_free_slp_instance (slp_instance, bool);
|
||||
extern void vect_free_slp_instance (slp_instance);
|
||||
extern bool vect_transform_slp_perm_load (vec_info *, slp_tree, vec<tree>,
|
||||
gimple_stmt_iterator *, poly_uint64,
|
||||
bool, unsigned *);
|
||||
|
|
@ -2047,6 +2043,7 @@ extern bool can_duplicate_and_interleave_p (vec_info *, unsigned int, tree,
|
|||
extern void duplicate_and_interleave (vec_info *, gimple_seq *, tree,
|
||||
vec<tree>, unsigned int, vec<tree> &);
|
||||
extern int vect_get_place_in_interleaving_chain (stmt_vec_info, stmt_vec_info);
|
||||
extern bool vect_update_shared_vectype (stmt_vec_info, tree);
|
||||
|
||||
/* In tree-vect-patterns.c. */
|
||||
/* Pattern recognition functions.
|
||||
|
|
|
|||
Loading…
Reference in New Issue