Refactor load/store costing
This passes down the already available alignment scheme and misalignment to the load/store costing routines, removing redundant queries. 2021-10-19 Richard Biener <rguenther@suse.de> * tree-vectorizer.h (vect_get_store_cost): Adjust signature. (vect_get_load_cost): Likewise. * tree-vect-data-refs.c (vect_get_data_access_cost): Get alignment support scheme and misalignment as arguments and pass them down. (vect_get_peeling_costs_all_drs): Compute that info here and note that we shouldn't need to. * tree-vect-stmts.c (vect_model_store_cost): Get alignment support scheme and misalignment as arguments. (vect_get_store_cost): Likewise. (vect_model_load_cost): Likewise. (vect_get_load_cost): Likewise. (vectorizable_store): Pass down alignment support scheme and misalignment to costing. (vectorizable_load): Likewise.
This commit is contained in:
parent
9890b12c72
commit
793d2549b1
|
@ -1396,7 +1396,9 @@ vector_alignment_reachable_p (dr_vec_info *dr_info)
|
||||||
|
|
||||||
static void
|
static void
|
||||||
vect_get_data_access_cost (vec_info *vinfo, dr_vec_info *dr_info,
|
vect_get_data_access_cost (vec_info *vinfo, dr_vec_info *dr_info,
|
||||||
unsigned int *inside_cost,
|
dr_alignment_support alignment_support_scheme,
|
||||||
|
int misalignment,
|
||||||
|
unsigned int *inside_cost,
|
||||||
unsigned int *outside_cost,
|
unsigned int *outside_cost,
|
||||||
stmt_vector_for_cost *body_cost_vec,
|
stmt_vector_for_cost *body_cost_vec,
|
||||||
stmt_vector_for_cost *prologue_cost_vec)
|
stmt_vector_for_cost *prologue_cost_vec)
|
||||||
|
@ -1411,10 +1413,12 @@ vect_get_data_access_cost (vec_info *vinfo, dr_vec_info *dr_info,
|
||||||
ncopies = vect_get_num_copies (loop_vinfo, STMT_VINFO_VECTYPE (stmt_info));
|
ncopies = vect_get_num_copies (loop_vinfo, STMT_VINFO_VECTYPE (stmt_info));
|
||||||
|
|
||||||
if (DR_IS_READ (dr_info->dr))
|
if (DR_IS_READ (dr_info->dr))
|
||||||
vect_get_load_cost (vinfo, stmt_info, ncopies, true, inside_cost,
|
vect_get_load_cost (vinfo, stmt_info, ncopies, alignment_support_scheme,
|
||||||
|
misalignment, true, inside_cost,
|
||||||
outside_cost, prologue_cost_vec, body_cost_vec, false);
|
outside_cost, prologue_cost_vec, body_cost_vec, false);
|
||||||
else
|
else
|
||||||
vect_get_store_cost (vinfo,stmt_info, ncopies, inside_cost, body_cost_vec);
|
vect_get_store_cost (vinfo,stmt_info, ncopies, alignment_support_scheme,
|
||||||
|
misalignment, inside_cost, body_cost_vec);
|
||||||
|
|
||||||
if (dump_enabled_p ())
|
if (dump_enabled_p ())
|
||||||
dump_printf_loc (MSG_NOTE, vect_location,
|
dump_printf_loc (MSG_NOTE, vect_location,
|
||||||
|
@ -1545,7 +1549,15 @@ vect_get_peeling_costs_all_drs (loop_vec_info loop_vinfo,
|
||||||
vect_dr_misalign_for_aligned_access (dr0_info));
|
vect_dr_misalign_for_aligned_access (dr0_info));
|
||||||
else
|
else
|
||||||
vect_update_misalignment_for_peel (dr_info, dr0_info, npeel);
|
vect_update_misalignment_for_peel (dr_info, dr0_info, npeel);
|
||||||
vect_get_data_access_cost (loop_vinfo, dr_info, inside_cost, outside_cost,
|
/* ??? We should be able to avoid both the adjustment before and the
|
||||||
|
call to vect_supportable_dr_alignment below. */
|
||||||
|
tree vectype = STMT_VINFO_VECTYPE (dr_info->stmt);
|
||||||
|
int misalignment = dr_misalignment (dr_info, vectype);
|
||||||
|
dr_alignment_support alignment_support_scheme
|
||||||
|
= vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype);
|
||||||
|
vect_get_data_access_cost (loop_vinfo, dr_info,
|
||||||
|
alignment_support_scheme, misalignment,
|
||||||
|
inside_cost, outside_cost,
|
||||||
body_cost_vec, prologue_cost_vec);
|
body_cost_vec, prologue_cost_vec);
|
||||||
SET_DR_MISALIGNMENT (dr_info, save_misalignment);
|
SET_DR_MISALIGNMENT (dr_info, save_misalignment);
|
||||||
}
|
}
|
||||||
|
|
|
@ -909,6 +909,8 @@ cfun_returns (tree decl)
|
||||||
static void
|
static void
|
||||||
vect_model_store_cost (vec_info *vinfo, stmt_vec_info stmt_info, int ncopies,
|
vect_model_store_cost (vec_info *vinfo, stmt_vec_info stmt_info, int ncopies,
|
||||||
vect_memory_access_type memory_access_type,
|
vect_memory_access_type memory_access_type,
|
||||||
|
dr_alignment_support alignment_support_scheme,
|
||||||
|
int misalignment,
|
||||||
vec_load_store_type vls_type, slp_tree slp_node,
|
vec_load_store_type vls_type, slp_tree slp_node,
|
||||||
stmt_vector_for_cost *cost_vec)
|
stmt_vector_for_cost *cost_vec)
|
||||||
{
|
{
|
||||||
|
@ -969,7 +971,8 @@ vect_model_store_cost (vec_info *vinfo, stmt_vec_info stmt_info, int ncopies,
|
||||||
scalar_store, stmt_info, 0, vect_body);
|
scalar_store, stmt_info, 0, vect_body);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
vect_get_store_cost (vinfo, stmt_info, ncopies, &inside_cost, cost_vec);
|
vect_get_store_cost (vinfo, stmt_info, ncopies, alignment_support_scheme,
|
||||||
|
misalignment, &inside_cost, cost_vec);
|
||||||
|
|
||||||
if (memory_access_type == VMAT_ELEMENTWISE
|
if (memory_access_type == VMAT_ELEMENTWISE
|
||||||
|| memory_access_type == VMAT_STRIDED_SLP)
|
|| memory_access_type == VMAT_STRIDED_SLP)
|
||||||
|
@ -1021,15 +1024,12 @@ vect_model_store_cost (vec_info *vinfo, stmt_vec_info stmt_info, int ncopies,
|
||||||
|
|
||||||
/* Calculate cost of DR's memory access. */
|
/* Calculate cost of DR's memory access. */
|
||||||
void
|
void
|
||||||
vect_get_store_cost (vec_info *vinfo, stmt_vec_info stmt_info, int ncopies,
|
vect_get_store_cost (vec_info *, stmt_vec_info stmt_info, int ncopies,
|
||||||
|
dr_alignment_support alignment_support_scheme,
|
||||||
|
int misalignment,
|
||||||
unsigned int *inside_cost,
|
unsigned int *inside_cost,
|
||||||
stmt_vector_for_cost *body_cost_vec)
|
stmt_vector_for_cost *body_cost_vec)
|
||||||
{
|
{
|
||||||
dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
|
|
||||||
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
|
|
||||||
dr_alignment_support alignment_support_scheme
|
|
||||||
= vect_supportable_dr_alignment (vinfo, dr_info, vectype);
|
|
||||||
|
|
||||||
switch (alignment_support_scheme)
|
switch (alignment_support_scheme)
|
||||||
{
|
{
|
||||||
case dr_aligned:
|
case dr_aligned:
|
||||||
|
@ -1049,8 +1049,7 @@ vect_get_store_cost (vec_info *vinfo, stmt_vec_info stmt_info, int ncopies,
|
||||||
/* Here, we assign an additional cost for the unaligned store. */
|
/* Here, we assign an additional cost for the unaligned store. */
|
||||||
*inside_cost += record_stmt_cost (body_cost_vec, ncopies,
|
*inside_cost += record_stmt_cost (body_cost_vec, ncopies,
|
||||||
unaligned_store, stmt_info,
|
unaligned_store, stmt_info,
|
||||||
dr_misalignment (dr_info, vectype),
|
misalignment, vect_body);
|
||||||
vect_body);
|
|
||||||
if (dump_enabled_p ())
|
if (dump_enabled_p ())
|
||||||
dump_printf_loc (MSG_NOTE, vect_location,
|
dump_printf_loc (MSG_NOTE, vect_location,
|
||||||
"vect_model_store_cost: unaligned supported by "
|
"vect_model_store_cost: unaligned supported by "
|
||||||
|
@ -1085,6 +1084,8 @@ static void
|
||||||
vect_model_load_cost (vec_info *vinfo,
|
vect_model_load_cost (vec_info *vinfo,
|
||||||
stmt_vec_info stmt_info, unsigned ncopies, poly_uint64 vf,
|
stmt_vec_info stmt_info, unsigned ncopies, poly_uint64 vf,
|
||||||
vect_memory_access_type memory_access_type,
|
vect_memory_access_type memory_access_type,
|
||||||
|
dr_alignment_support alignment_support_scheme,
|
||||||
|
int misalignment,
|
||||||
gather_scatter_info *gs_info,
|
gather_scatter_info *gs_info,
|
||||||
slp_tree slp_node,
|
slp_tree slp_node,
|
||||||
stmt_vector_for_cost *cost_vec)
|
stmt_vector_for_cost *cost_vec)
|
||||||
|
@ -1144,7 +1145,8 @@ vect_model_load_cost (vec_info *vinfo,
|
||||||
dump_printf_loc (MSG_NOTE, vect_location,
|
dump_printf_loc (MSG_NOTE, vect_location,
|
||||||
"vect_model_load_cost: %d unused vectors.\n",
|
"vect_model_load_cost: %d unused vectors.\n",
|
||||||
gaps);
|
gaps);
|
||||||
vect_get_load_cost (vinfo, stmt_info, ncopies * gaps, false,
|
vect_get_load_cost (vinfo, stmt_info, ncopies * gaps,
|
||||||
|
alignment_support_scheme, misalignment, false,
|
||||||
&inside_cost, &prologue_cost,
|
&inside_cost, &prologue_cost,
|
||||||
cost_vec, cost_vec, true);
|
cost_vec, cost_vec, true);
|
||||||
}
|
}
|
||||||
|
@ -1190,7 +1192,8 @@ vect_model_load_cost (vec_info *vinfo,
|
||||||
scalar_load, stmt_info, 0, vect_body);
|
scalar_load, stmt_info, 0, vect_body);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
vect_get_load_cost (vinfo, stmt_info, ncopies, first_stmt_p,
|
vect_get_load_cost (vinfo, stmt_info, ncopies,
|
||||||
|
alignment_support_scheme, misalignment, first_stmt_p,
|
||||||
&inside_cost, &prologue_cost,
|
&inside_cost, &prologue_cost,
|
||||||
cost_vec, cost_vec, true);
|
cost_vec, cost_vec, true);
|
||||||
if (memory_access_type == VMAT_ELEMENTWISE
|
if (memory_access_type == VMAT_ELEMENTWISE
|
||||||
|
@ -1209,18 +1212,15 @@ vect_model_load_cost (vec_info *vinfo,
|
||||||
|
|
||||||
/* Calculate cost of DR's memory access. */
|
/* Calculate cost of DR's memory access. */
|
||||||
void
|
void
|
||||||
vect_get_load_cost (vec_info *vinfo, stmt_vec_info stmt_info, int ncopies,
|
vect_get_load_cost (vec_info *, stmt_vec_info stmt_info, int ncopies,
|
||||||
|
dr_alignment_support alignment_support_scheme,
|
||||||
|
int misalignment,
|
||||||
bool add_realign_cost, unsigned int *inside_cost,
|
bool add_realign_cost, unsigned int *inside_cost,
|
||||||
unsigned int *prologue_cost,
|
unsigned int *prologue_cost,
|
||||||
stmt_vector_for_cost *prologue_cost_vec,
|
stmt_vector_for_cost *prologue_cost_vec,
|
||||||
stmt_vector_for_cost *body_cost_vec,
|
stmt_vector_for_cost *body_cost_vec,
|
||||||
bool record_prologue_costs)
|
bool record_prologue_costs)
|
||||||
{
|
{
|
||||||
dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
|
|
||||||
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
|
|
||||||
dr_alignment_support alignment_support_scheme
|
|
||||||
= vect_supportable_dr_alignment (vinfo, dr_info, vectype);
|
|
||||||
|
|
||||||
switch (alignment_support_scheme)
|
switch (alignment_support_scheme)
|
||||||
{
|
{
|
||||||
case dr_aligned:
|
case dr_aligned:
|
||||||
|
@ -1239,8 +1239,7 @@ vect_get_load_cost (vec_info *vinfo, stmt_vec_info stmt_info, int ncopies,
|
||||||
/* Here, we assign an additional cost for the unaligned load. */
|
/* Here, we assign an additional cost for the unaligned load. */
|
||||||
*inside_cost += record_stmt_cost (body_cost_vec, ncopies,
|
*inside_cost += record_stmt_cost (body_cost_vec, ncopies,
|
||||||
unaligned_load, stmt_info,
|
unaligned_load, stmt_info,
|
||||||
dr_misalignment (dr_info, vectype),
|
misalignment, vect_body);
|
||||||
vect_body);
|
|
||||||
|
|
||||||
if (dump_enabled_p ())
|
if (dump_enabled_p ())
|
||||||
dump_printf_loc (MSG_NOTE, vect_location,
|
dump_printf_loc (MSG_NOTE, vect_location,
|
||||||
|
@ -7443,7 +7442,8 @@ vectorizable_store (vec_info *vinfo,
|
||||||
|
|
||||||
STMT_VINFO_TYPE (stmt_info) = store_vec_info_type;
|
STMT_VINFO_TYPE (stmt_info) = store_vec_info_type;
|
||||||
vect_model_store_cost (vinfo, stmt_info, ncopies,
|
vect_model_store_cost (vinfo, stmt_info, ncopies,
|
||||||
memory_access_type, vls_type, slp_node, cost_vec);
|
memory_access_type, alignment_support_scheme,
|
||||||
|
misalignment, vls_type, slp_node, cost_vec);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
gcc_assert (memory_access_type == STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info));
|
gcc_assert (memory_access_type == STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info));
|
||||||
|
@ -8779,6 +8779,7 @@ vectorizable_load (vec_info *vinfo,
|
||||||
|
|
||||||
STMT_VINFO_TYPE (stmt_info) = load_vec_info_type;
|
STMT_VINFO_TYPE (stmt_info) = load_vec_info_type;
|
||||||
vect_model_load_cost (vinfo, stmt_info, ncopies, vf, memory_access_type,
|
vect_model_load_cost (vinfo, stmt_info, ncopies, vf, memory_access_type,
|
||||||
|
alignment_support_scheme, misalignment,
|
||||||
&gs_info, slp_node, cost_vec);
|
&gs_info, slp_node, cost_vec);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
|
@ -1957,11 +1957,13 @@ extern bool vect_nop_conversion_p (stmt_vec_info);
|
||||||
extern opt_result vect_analyze_stmt (vec_info *, stmt_vec_info, bool *,
|
extern opt_result vect_analyze_stmt (vec_info *, stmt_vec_info, bool *,
|
||||||
slp_tree,
|
slp_tree,
|
||||||
slp_instance, stmt_vector_for_cost *);
|
slp_instance, stmt_vector_for_cost *);
|
||||||
extern void vect_get_load_cost (vec_info *, stmt_vec_info, int, bool,
|
extern void vect_get_load_cost (vec_info *, stmt_vec_info, int,
|
||||||
|
dr_alignment_support, int, bool,
|
||||||
unsigned int *, unsigned int *,
|
unsigned int *, unsigned int *,
|
||||||
stmt_vector_for_cost *,
|
stmt_vector_for_cost *,
|
||||||
stmt_vector_for_cost *, bool);
|
stmt_vector_for_cost *, bool);
|
||||||
extern void vect_get_store_cost (vec_info *, stmt_vec_info, int,
|
extern void vect_get_store_cost (vec_info *, stmt_vec_info, int,
|
||||||
|
dr_alignment_support, int,
|
||||||
unsigned int *, stmt_vector_for_cost *);
|
unsigned int *, stmt_vector_for_cost *);
|
||||||
extern bool vect_supportable_shift (vec_info *, enum tree_code, tree);
|
extern bool vect_supportable_shift (vec_info *, enum tree_code, tree);
|
||||||
extern tree vect_gen_perm_mask_any (tree, const vec_perm_indices &);
|
extern tree vect_gen_perm_mask_any (tree, const vec_perm_indices &);
|
||||||
|
|
Loading…
Reference in New Issue