2041 lines
66 KiB
C
2041 lines
66 KiB
C
/* Transformation Utilities for Loop Vectorization.
|
|
Copyright (C) 2003,2004,2005 Free Software Foundation, Inc.
|
|
Contributed by Dorit Naishlos <dorit@il.ibm.com>
|
|
|
|
This file is part of GCC.
|
|
|
|
GCC is free software; you can redistribute it and/or modify it under
|
|
the terms of the GNU General Public License as published by the Free
|
|
Software Foundation; either version 2, or (at your option) any later
|
|
version.
|
|
|
|
GCC is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
for more details.
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
along with GCC; see the file COPYING. If not, write to the Free
|
|
Software Foundation, 59 Temple Place - Suite 330, Boston, MA
|
|
02111-1307, USA. */
|
|
|
|
#include "config.h"
|
|
#include "system.h"
|
|
#include "coretypes.h"
|
|
#include "tm.h"
|
|
#include "errors.h"
|
|
#include "ggc.h"
|
|
#include "tree.h"
|
|
#include "target.h"
|
|
#include "rtl.h"
|
|
#include "basic-block.h"
|
|
#include "diagnostic.h"
|
|
#include "tree-flow.h"
|
|
#include "tree-dump.h"
|
|
#include "timevar.h"
|
|
#include "cfgloop.h"
|
|
#include "expr.h"
|
|
#include "optabs.h"
|
|
#include "tree-data-ref.h"
|
|
#include "tree-chrec.h"
|
|
#include "tree-scalar-evolution.h"
|
|
#include "tree-vectorizer.h"
|
|
#include "langhooks.h"
|
|
#include "tree-pass.h"
|
|
#include "toplev.h"
|
|
|
|
/* Utility functions for the code transformation. */
|
|
static bool vect_transform_stmt (tree, block_stmt_iterator *);
|
|
static void vect_align_data_ref (tree);
|
|
static tree vect_create_destination_var (tree, tree);
|
|
static tree vect_create_data_ref_ptr
|
|
(tree, block_stmt_iterator *, tree, tree *, bool);
|
|
static tree vect_create_index_for_vector_ref (loop_vec_info);
|
|
static tree vect_create_addr_base_for_vector_ref (tree, tree *, tree);
|
|
static tree vect_get_new_vect_var (tree, enum vect_var_kind, const char *);
|
|
static tree vect_get_vec_def_for_operand (tree, tree);
|
|
static tree vect_init_vector (tree, tree);
|
|
static void vect_finish_stmt_generation
|
|
(tree stmt, tree vec_stmt, block_stmt_iterator *bsi);
|
|
static bool vect_is_simple_cond (tree, loop_vec_info);
|
|
static void update_vuses_to_preheader (tree, struct loop*);
|
|
|
|
/* Utility function dealing with loop peeling (not peeling itself). */
|
|
static void vect_generate_tmps_on_preheader
|
|
(loop_vec_info, tree *, tree *, tree *);
|
|
static tree vect_build_loop_niters (loop_vec_info);
|
|
static void vect_update_ivs_after_vectorizer (loop_vec_info, tree, edge);
|
|
static tree vect_gen_niters_for_prolog_loop (loop_vec_info, tree);
|
|
static void vect_update_init_of_dr (struct data_reference *, tree niters);
|
|
static void vect_update_inits_of_drs (loop_vec_info, tree);
|
|
static void vect_do_peeling_for_alignment (loop_vec_info, struct loops *);
|
|
static void vect_do_peeling_for_loop_bound
|
|
(loop_vec_info, tree *, struct loops *);
|
|
|
|
|
|
/* Function vect_get_new_vect_var.
|
|
|
|
Returns a name for a new variable. The current naming scheme appends the
|
|
prefix "vect_" or "vect_p" (depending on the value of VAR_KIND) to
|
|
the name of vectorizer generated variables, and appends that to NAME if
|
|
provided. */
|
|
|
|
static tree
|
|
vect_get_new_vect_var (tree type, enum vect_var_kind var_kind, const char *name)
|
|
{
|
|
const char *prefix;
|
|
tree new_vect_var;
|
|
|
|
if (var_kind == vect_simple_var)
|
|
prefix = "vect_";
|
|
else
|
|
prefix = "vect_p";
|
|
|
|
if (name)
|
|
new_vect_var = create_tmp_var (type, concat (prefix, name, NULL));
|
|
else
|
|
new_vect_var = create_tmp_var (type, prefix);
|
|
|
|
return new_vect_var;
|
|
}
|
|
|
|
|
|
/* Function vect_create_index_for_vector_ref.
|
|
|
|
Create (and return) an index variable, along with it's update chain in the
|
|
loop. This variable will be used to access a memory location in a vector
|
|
operation.
|
|
|
|
Input:
|
|
LOOP: The loop being vectorized.
|
|
BSI: The block_stmt_iterator where STMT is. Any new stmts created by this
|
|
function can be added here, or in the loop pre-header.
|
|
|
|
Output:
|
|
Return an index that will be used to index a vector array. It is expected
|
|
that a pointer to the first vector will be used as the base address for the
|
|
indexed reference.
|
|
|
|
FORNOW: we are not trying to be efficient, just creating a new index each
|
|
time from scratch. At this time all vector references could use the same
|
|
index.
|
|
|
|
TODO: create only one index to be used by all vector references. Record
|
|
the index in the LOOP_VINFO the first time this procedure is called and
|
|
return it on subsequent calls. The increment of this index must be placed
|
|
just before the conditional expression that ends the single block loop. */
|
|
|
|
static tree
|
|
vect_create_index_for_vector_ref (loop_vec_info loop_vinfo)
|
|
{
|
|
tree init, step;
|
|
block_stmt_iterator incr_bsi;
|
|
bool insert_after;
|
|
tree indx_before_incr, indx_after_incr;
|
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
|
tree incr;
|
|
|
|
/* It is assumed that the base pointer used for vectorized access contains
|
|
the address of the first vector. Therefore the index used for vectorized
|
|
access must be initialized to zero and incremented by 1. */
|
|
|
|
init = integer_zero_node;
|
|
step = integer_one_node;
|
|
|
|
standard_iv_increment_position (loop, &incr_bsi, &insert_after);
|
|
create_iv (init, step, NULL_TREE, loop, &incr_bsi, insert_after,
|
|
&indx_before_incr, &indx_after_incr);
|
|
incr = bsi_stmt (incr_bsi);
|
|
set_stmt_info (stmt_ann (incr), new_stmt_vec_info (incr, loop_vinfo));
|
|
|
|
return indx_before_incr;
|
|
}
|
|
|
|
|
|
/* Function vect_create_addr_base_for_vector_ref.
|
|
|
|
Create an expression that computes the address of the first memory location
|
|
that will be accessed for a data reference.
|
|
|
|
Input:
|
|
STMT: The statement containing the data reference.
|
|
NEW_STMT_LIST: Must be initialized to NULL_TREE or a statement list.
|
|
OFFSET: Optional. If supplied, it is be added to the initial address.
|
|
|
|
Output:
|
|
1. Return an SSA_NAME whose value is the address of the memory location of
|
|
the first vector of the data reference.
|
|
2. If new_stmt_list is not NULL_TREE after return then the caller must insert
|
|
these statement(s) which define the returned SSA_NAME.
|
|
|
|
FORNOW: We are only handling array accesses with step 1. */
|
|
|
|
static tree
|
|
vect_create_addr_base_for_vector_ref (tree stmt,
|
|
tree *new_stmt_list,
|
|
tree offset)
|
|
{
|
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
|
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
|
|
tree data_ref_base =
|
|
unshare_expr (STMT_VINFO_VECT_DR_BASE_ADDRESS (stmt_info));
|
|
tree base_name = build_fold_indirect_ref (data_ref_base);
|
|
tree ref = DR_REF (dr);
|
|
tree scalar_type = TREE_TYPE (ref);
|
|
tree scalar_ptr_type = build_pointer_type (scalar_type);
|
|
tree vec_stmt;
|
|
tree new_temp;
|
|
tree addr_base, addr_expr;
|
|
tree dest, new_stmt;
|
|
tree base_offset = unshare_expr (STMT_VINFO_VECT_INIT_OFFSET (stmt_info));
|
|
|
|
/* Create base_offset */
|
|
dest = create_tmp_var (TREE_TYPE (base_offset), "base_off");
|
|
add_referenced_tmp_var (dest);
|
|
base_offset = force_gimple_operand (base_offset, &new_stmt, false, dest);
|
|
append_to_statement_list_force (new_stmt, new_stmt_list);
|
|
|
|
if (offset)
|
|
{
|
|
tree tmp = create_tmp_var (TREE_TYPE (base_offset), "offset");
|
|
add_referenced_tmp_var (tmp);
|
|
offset = fold (build2 (MULT_EXPR, TREE_TYPE (offset), offset,
|
|
STMT_VINFO_VECT_STEP (stmt_info)));
|
|
base_offset = fold (build2 (PLUS_EXPR, TREE_TYPE (base_offset),
|
|
base_offset, offset));
|
|
base_offset = force_gimple_operand (base_offset, &new_stmt, false, tmp);
|
|
append_to_statement_list_force (new_stmt, new_stmt_list);
|
|
}
|
|
|
|
/* base + base_offset */
|
|
addr_base = fold (build2 (PLUS_EXPR, TREE_TYPE (data_ref_base), data_ref_base,
|
|
base_offset));
|
|
|
|
/* addr_expr = addr_base */
|
|
addr_expr = vect_get_new_vect_var (scalar_ptr_type, vect_pointer_var,
|
|
get_name (base_name));
|
|
add_referenced_tmp_var (addr_expr);
|
|
vec_stmt = build2 (MODIFY_EXPR, void_type_node, addr_expr, addr_base);
|
|
new_temp = make_ssa_name (addr_expr, vec_stmt);
|
|
TREE_OPERAND (vec_stmt, 0) = new_temp;
|
|
append_to_statement_list_force (vec_stmt, new_stmt_list);
|
|
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
{
|
|
fprintf (vect_dump, "created ");
|
|
print_generic_expr (vect_dump, vec_stmt, TDF_SLIM);
|
|
}
|
|
return new_temp;
|
|
}
|
|
|
|
|
|
/* Function vect_align_data_ref.
|
|
|
|
Handle misalignment of a memory accesses.
|
|
|
|
FORNOW: Can't handle misaligned accesses.
|
|
Make sure that the dataref is aligned. */
|
|
|
|
static void
|
|
vect_align_data_ref (tree stmt)
|
|
{
|
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
|
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
|
|
|
|
/* FORNOW: can't handle misaligned accesses;
|
|
all accesses expected to be aligned. */
|
|
gcc_assert (aligned_access_p (dr));
|
|
}
|
|
|
|
|
|
/* Function vect_create_data_ref_ptr.
|
|
|
|
Create a memory reference expression for vector access, to be used in a
|
|
vector load/store stmt. The reference is based on a new pointer to vector
|
|
type (vp).
|
|
|
|
Input:
|
|
1. STMT: a stmt that references memory. Expected to be of the form
|
|
MODIFY_EXPR <name, data-ref> or MODIFY_EXPR <data-ref, name>.
|
|
2. BSI: block_stmt_iterator where new stmts can be added.
|
|
3. OFFSET (optional): an offset to be added to the initial address accessed
|
|
by the data-ref in STMT.
|
|
4. ONLY_INIT: indicate if vp is to be updated in the loop, or remain
|
|
pointing to the initial address.
|
|
|
|
Output:
|
|
1. Declare a new ptr to vector_type, and have it point to the base of the
|
|
data reference (initial addressed accessed by the data reference).
|
|
For example, for vector of type V8HI, the following code is generated:
|
|
|
|
v8hi *vp;
|
|
vp = (v8hi *)initial_address;
|
|
|
|
if OFFSET is not supplied:
|
|
initial_address = &a[init];
|
|
if OFFSET is supplied:
|
|
initial_address = &a[init + OFFSET];
|
|
|
|
Return the initial_address in INITIAL_ADDRESS.
|
|
|
|
2. Create a data-reference in the loop based on the new vector pointer vp,
|
|
and using a new index variable 'idx' as follows:
|
|
|
|
vp' = vp + update
|
|
|
|
where if ONLY_INIT is true:
|
|
update = zero
|
|
and otherwise
|
|
update = idx + vector_type_size
|
|
|
|
Return the pointer vp'.
|
|
|
|
|
|
FORNOW: handle only aligned and consecutive accesses. */
|
|
|
|
static tree
|
|
vect_create_data_ref_ptr (tree stmt, block_stmt_iterator *bsi, tree offset,
|
|
tree *initial_address, bool only_init)
|
|
{
|
|
tree base_name;
|
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
|
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
|
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
|
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
|
|
tree vect_ptr_type;
|
|
tree vect_ptr;
|
|
tree tag;
|
|
tree new_temp;
|
|
tree vec_stmt;
|
|
tree new_stmt_list = NULL_TREE;
|
|
tree idx;
|
|
edge pe = loop_preheader_edge (loop);
|
|
basic_block new_bb;
|
|
tree vect_ptr_init;
|
|
tree vectype_size;
|
|
tree ptr_update;
|
|
tree data_ref_ptr;
|
|
tree type, tmp, size;
|
|
|
|
base_name = build_fold_indirect_ref (unshare_expr (
|
|
STMT_VINFO_VECT_DR_BASE_ADDRESS (stmt_info)));
|
|
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
{
|
|
tree data_ref_base = base_name;
|
|
fprintf (vect_dump, "create array_ref of type: ");
|
|
print_generic_expr (vect_dump, vectype, TDF_SLIM);
|
|
if (TREE_CODE (data_ref_base) == VAR_DECL)
|
|
fprintf (vect_dump, " vectorizing a one dimensional array ref: ");
|
|
else if (TREE_CODE (data_ref_base) == ARRAY_REF)
|
|
fprintf (vect_dump, " vectorizing a multidimensional array ref: ");
|
|
else if (TREE_CODE (data_ref_base) == COMPONENT_REF)
|
|
fprintf (vect_dump, " vectorizing a record based array ref: ");
|
|
else if (TREE_CODE (data_ref_base) == SSA_NAME)
|
|
fprintf (vect_dump, " vectorizing a pointer ref: ");
|
|
print_generic_expr (vect_dump, base_name, TDF_SLIM);
|
|
}
|
|
|
|
/** (1) Create the new vector-pointer variable: **/
|
|
|
|
vect_ptr_type = build_pointer_type (vectype);
|
|
vect_ptr = vect_get_new_vect_var (vect_ptr_type, vect_pointer_var,
|
|
get_name (base_name));
|
|
add_referenced_tmp_var (vect_ptr);
|
|
|
|
|
|
/** (2) Add aliasing information to the new vector-pointer:
|
|
(The points-to info (SSA_NAME_PTR_INFO) may be defined later.) **/
|
|
|
|
tag = STMT_VINFO_MEMTAG (stmt_info);
|
|
gcc_assert (tag);
|
|
|
|
/* If the memory tag of the original reference was not a type tag or
|
|
if the pointed-to type of VECT_PTR has an alias set number
|
|
different than TAG's, then we need to create a new type tag for
|
|
VECT_PTR and add TAG to its alias set. */
|
|
if (var_ann (tag)->mem_tag_kind == NOT_A_TAG
|
|
|| get_alias_set (tag) != get_alias_set (TREE_TYPE (vect_ptr_type)))
|
|
add_type_alias (vect_ptr, tag);
|
|
else
|
|
var_ann (vect_ptr)->type_mem_tag = tag;
|
|
|
|
var_ann (vect_ptr)->subvars = STMT_VINFO_SUBVARS (stmt_info);
|
|
|
|
/** (3) Calculate the initial address the vector-pointer, and set
|
|
the vector-pointer to point to it before the loop: **/
|
|
|
|
/* Create: (&(base[init_val+offset]) in the loop preheader. */
|
|
new_temp = vect_create_addr_base_for_vector_ref (stmt, &new_stmt_list,
|
|
offset);
|
|
pe = loop_preheader_edge (loop);
|
|
new_bb = bsi_insert_on_edge_immediate (pe, new_stmt_list);
|
|
gcc_assert (!new_bb);
|
|
*initial_address = new_temp;
|
|
|
|
/* Create: p = (vectype *) initial_base */
|
|
vec_stmt = fold_convert (vect_ptr_type, new_temp);
|
|
vec_stmt = build2 (MODIFY_EXPR, void_type_node, vect_ptr, vec_stmt);
|
|
new_temp = make_ssa_name (vect_ptr, vec_stmt);
|
|
TREE_OPERAND (vec_stmt, 0) = new_temp;
|
|
new_bb = bsi_insert_on_edge_immediate (pe, vec_stmt);
|
|
gcc_assert (!new_bb);
|
|
vect_ptr_init = TREE_OPERAND (vec_stmt, 0);
|
|
|
|
|
|
/** (4) Handle the updating of the vector-pointer inside the loop: **/
|
|
|
|
if (only_init) /* No update in loop is required. */
|
|
{
|
|
/* Copy the points-to information if it exists. */
|
|
if (STMT_VINFO_PTR_INFO (stmt_info))
|
|
duplicate_ssa_name_ptr_info (vect_ptr_init,
|
|
STMT_VINFO_PTR_INFO (stmt_info));
|
|
return vect_ptr_init;
|
|
}
|
|
|
|
idx = vect_create_index_for_vector_ref (loop_vinfo);
|
|
|
|
/* Create: update = idx * vectype_size */
|
|
tmp = create_tmp_var (integer_type_node, "update");
|
|
add_referenced_tmp_var (tmp);
|
|
size = TYPE_SIZE (vect_ptr_type);
|
|
type = lang_hooks.types.type_for_size (tree_low_cst (size, 1), 1);
|
|
ptr_update = create_tmp_var (type, "update");
|
|
add_referenced_tmp_var (ptr_update);
|
|
vectype_size = TYPE_SIZE_UNIT (vectype);
|
|
vec_stmt = build2 (MULT_EXPR, integer_type_node, idx, vectype_size);
|
|
vec_stmt = build2 (MODIFY_EXPR, void_type_node, tmp, vec_stmt);
|
|
new_temp = make_ssa_name (tmp, vec_stmt);
|
|
TREE_OPERAND (vec_stmt, 0) = new_temp;
|
|
bsi_insert_before (bsi, vec_stmt, BSI_SAME_STMT);
|
|
vec_stmt = fold_convert (type, new_temp);
|
|
vec_stmt = build2 (MODIFY_EXPR, void_type_node, ptr_update, vec_stmt);
|
|
new_temp = make_ssa_name (ptr_update, vec_stmt);
|
|
TREE_OPERAND (vec_stmt, 0) = new_temp;
|
|
bsi_insert_before (bsi, vec_stmt, BSI_SAME_STMT);
|
|
|
|
/* Create: data_ref_ptr = vect_ptr_init + update */
|
|
vec_stmt = build2 (PLUS_EXPR, vect_ptr_type, vect_ptr_init, new_temp);
|
|
vec_stmt = build2 (MODIFY_EXPR, void_type_node, vect_ptr, vec_stmt);
|
|
new_temp = make_ssa_name (vect_ptr, vec_stmt);
|
|
TREE_OPERAND (vec_stmt, 0) = new_temp;
|
|
bsi_insert_before (bsi, vec_stmt, BSI_SAME_STMT);
|
|
data_ref_ptr = TREE_OPERAND (vec_stmt, 0);
|
|
|
|
/* Copy the points-to information if it exists. */
|
|
if (STMT_VINFO_PTR_INFO (stmt_info))
|
|
duplicate_ssa_name_ptr_info (data_ref_ptr, STMT_VINFO_PTR_INFO (stmt_info));
|
|
return data_ref_ptr;
|
|
}
|
|
|
|
|
|
/* Function vect_create_destination_var.
|
|
|
|
Create a new temporary of type VECTYPE. */
|
|
|
|
static tree
|
|
vect_create_destination_var (tree scalar_dest, tree vectype)
|
|
{
|
|
tree vec_dest;
|
|
const char *new_name;
|
|
|
|
gcc_assert (TREE_CODE (scalar_dest) == SSA_NAME);
|
|
|
|
new_name = get_name (scalar_dest);
|
|
if (!new_name)
|
|
new_name = "var_";
|
|
vec_dest = vect_get_new_vect_var (vectype, vect_simple_var, new_name);
|
|
add_referenced_tmp_var (vec_dest);
|
|
|
|
return vec_dest;
|
|
}
|
|
|
|
|
|
/* Function vect_init_vector.
|
|
|
|
Insert a new stmt (INIT_STMT) that initializes a new vector variable with
|
|
the vector elements of VECTOR_VAR. Return the DEF of INIT_STMT. It will be
|
|
used in the vectorization of STMT. */
|
|
|
|
static tree
|
|
vect_init_vector (tree stmt, tree vector_var)
|
|
{
|
|
stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
|
|
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
|
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
|
tree new_var;
|
|
tree init_stmt;
|
|
tree vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
|
|
tree vec_oprnd;
|
|
edge pe;
|
|
tree new_temp;
|
|
basic_block new_bb;
|
|
|
|
new_var = vect_get_new_vect_var (vectype, vect_simple_var, "cst_");
|
|
add_referenced_tmp_var (new_var);
|
|
|
|
init_stmt = build2 (MODIFY_EXPR, vectype, new_var, vector_var);
|
|
new_temp = make_ssa_name (new_var, init_stmt);
|
|
TREE_OPERAND (init_stmt, 0) = new_temp;
|
|
|
|
pe = loop_preheader_edge (loop);
|
|
new_bb = bsi_insert_on_edge_immediate (pe, init_stmt);
|
|
gcc_assert (!new_bb);
|
|
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
{
|
|
fprintf (vect_dump, "created new init_stmt: ");
|
|
print_generic_expr (vect_dump, init_stmt, TDF_SLIM);
|
|
}
|
|
|
|
vec_oprnd = TREE_OPERAND (init_stmt, 0);
|
|
return vec_oprnd;
|
|
}
|
|
|
|
|
|
/* Function vect_get_vec_def_for_operand.
|
|
|
|
OP is an operand in STMT. This function returns a (vector) def that will be
|
|
used in the vectorized stmt for STMT.
|
|
|
|
In the case that OP is an SSA_NAME which is defined in the loop, then
|
|
STMT_VINFO_VEC_STMT of the defining stmt holds the relevant def.
|
|
|
|
In case OP is an invariant or constant, a new stmt that creates a vector def
|
|
needs to be introduced. */
|
|
|
|
static tree
|
|
vect_get_vec_def_for_operand (tree op, tree stmt)
|
|
{
|
|
tree vec_oprnd;
|
|
tree vec_stmt;
|
|
tree def_stmt;
|
|
stmt_vec_info def_stmt_info = NULL;
|
|
stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
|
|
tree vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
|
|
int nunits = TYPE_VECTOR_SUBPARTS (vectype);
|
|
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
|
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
|
basic_block bb;
|
|
tree vec_inv;
|
|
tree t = NULL_TREE;
|
|
tree def;
|
|
int i;
|
|
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
{
|
|
fprintf (vect_dump, "vect_get_vec_def_for_operand: ");
|
|
print_generic_expr (vect_dump, op, TDF_SLIM);
|
|
}
|
|
|
|
/** ===> Case 1: operand is a constant. **/
|
|
|
|
if (TREE_CODE (op) == INTEGER_CST || TREE_CODE (op) == REAL_CST)
|
|
{
|
|
/* Create 'vect_cst_ = {cst,cst,...,cst}' */
|
|
|
|
tree vec_cst;
|
|
|
|
/* Build a tree with vector elements. */
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
fprintf (vect_dump, "Create vector_cst. nunits = %d", nunits);
|
|
|
|
for (i = nunits - 1; i >= 0; --i)
|
|
{
|
|
t = tree_cons (NULL_TREE, op, t);
|
|
}
|
|
vec_cst = build_vector (vectype, t);
|
|
return vect_init_vector (stmt, vec_cst);
|
|
}
|
|
|
|
gcc_assert (TREE_CODE (op) == SSA_NAME);
|
|
|
|
/** ===> Case 2: operand is an SSA_NAME - find the stmt that defines it. **/
|
|
|
|
def_stmt = SSA_NAME_DEF_STMT (op);
|
|
def_stmt_info = vinfo_for_stmt (def_stmt);
|
|
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
{
|
|
fprintf (vect_dump, "vect_get_vec_def_for_operand: def_stmt: ");
|
|
print_generic_expr (vect_dump, def_stmt, TDF_SLIM);
|
|
}
|
|
|
|
|
|
/** ==> Case 2.1: operand is defined inside the loop. **/
|
|
|
|
if (def_stmt_info)
|
|
{
|
|
/* Get the def from the vectorized stmt. */
|
|
|
|
vec_stmt = STMT_VINFO_VEC_STMT (def_stmt_info);
|
|
gcc_assert (vec_stmt);
|
|
vec_oprnd = TREE_OPERAND (vec_stmt, 0);
|
|
return vec_oprnd;
|
|
}
|
|
|
|
|
|
/** ==> Case 2.2: operand is defined by the loop-header phi-node -
|
|
it is a reduction/induction. **/
|
|
|
|
bb = bb_for_stmt (def_stmt);
|
|
if (TREE_CODE (def_stmt) == PHI_NODE && flow_bb_inside_loop_p (loop, bb))
|
|
{
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
fprintf (vect_dump, "reduction/induction - unsupported.");
|
|
internal_error ("no support for reduction/induction"); /* FORNOW */
|
|
}
|
|
|
|
|
|
/** ==> Case 2.3: operand is defined outside the loop -
|
|
it is a loop invariant. */
|
|
|
|
switch (TREE_CODE (def_stmt))
|
|
{
|
|
case PHI_NODE:
|
|
def = PHI_RESULT (def_stmt);
|
|
break;
|
|
case MODIFY_EXPR:
|
|
def = TREE_OPERAND (def_stmt, 0);
|
|
break;
|
|
case NOP_EXPR:
|
|
def = TREE_OPERAND (def_stmt, 0);
|
|
gcc_assert (IS_EMPTY_STMT (def_stmt));
|
|
def = op;
|
|
break;
|
|
default:
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
{
|
|
fprintf (vect_dump, "unsupported defining stmt: ");
|
|
print_generic_expr (vect_dump, def_stmt, TDF_SLIM);
|
|
}
|
|
internal_error ("unsupported defining stmt");
|
|
}
|
|
|
|
/* Build a tree with vector elements.
|
|
Create 'vec_inv = {inv,inv,..,inv}' */
|
|
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
fprintf (vect_dump, "Create vector_inv.");
|
|
|
|
for (i = nunits - 1; i >= 0; --i)
|
|
{
|
|
t = tree_cons (NULL_TREE, def, t);
|
|
}
|
|
|
|
vec_inv = build_constructor (vectype, t);
|
|
return vect_init_vector (stmt, vec_inv);
|
|
}
|
|
|
|
|
|
/* Function vect_finish_stmt_generation.
|
|
|
|
Insert a new stmt. */
|
|
|
|
static void
|
|
vect_finish_stmt_generation (tree stmt, tree vec_stmt, block_stmt_iterator *bsi)
|
|
{
|
|
bsi_insert_before (bsi, vec_stmt, BSI_SAME_STMT);
|
|
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
{
|
|
fprintf (vect_dump, "add new stmt: ");
|
|
print_generic_expr (vect_dump, vec_stmt, TDF_SLIM);
|
|
}
|
|
|
|
#ifdef ENABLE_CHECKING
|
|
/* Make sure bsi points to the stmt that is being vectorized. */
|
|
gcc_assert (stmt == bsi_stmt (*bsi));
|
|
#endif
|
|
|
|
#ifdef USE_MAPPED_LOCATION
|
|
SET_EXPR_LOCATION (vec_stmt, EXPR_LOCATION (stmt));
|
|
#else
|
|
SET_EXPR_LOCUS (vec_stmt, EXPR_LOCUS (stmt));
|
|
#endif
|
|
}
|
|
|
|
|
|
/* Function vectorizable_assignment.
|
|
|
|
Check if STMT performs an assignment (copy) that can be vectorized.
|
|
If VEC_STMT is also passed, vectorize the STMT: create a vectorized
|
|
stmt to replace it, put it in VEC_STMT, and insert it at BSI.
|
|
Return FALSE if not a vectorizable STMT, TRUE otherwise. */
|
|
|
|
bool
|
|
vectorizable_assignment (tree stmt, block_stmt_iterator *bsi, tree *vec_stmt)
|
|
{
|
|
tree vec_dest;
|
|
tree scalar_dest;
|
|
tree op;
|
|
tree vec_oprnd;
|
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
|
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
|
|
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
|
|
tree new_temp;
|
|
|
|
/* Is vectorizable assignment? */
|
|
|
|
if (TREE_CODE (stmt) != MODIFY_EXPR)
|
|
return false;
|
|
|
|
scalar_dest = TREE_OPERAND (stmt, 0);
|
|
if (TREE_CODE (scalar_dest) != SSA_NAME)
|
|
return false;
|
|
|
|
op = TREE_OPERAND (stmt, 1);
|
|
if (!vect_is_simple_use (op, loop_vinfo, NULL))
|
|
{
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
fprintf (vect_dump, "use not simple.");
|
|
return false;
|
|
}
|
|
|
|
if (!vec_stmt) /* transformation not required. */
|
|
{
|
|
STMT_VINFO_TYPE (stmt_info) = assignment_vec_info_type;
|
|
return true;
|
|
}
|
|
|
|
/** Transform. **/
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
fprintf (vect_dump, "transform assignment.");
|
|
|
|
/* Handle def. */
|
|
vec_dest = vect_create_destination_var (scalar_dest, vectype);
|
|
|
|
/* Handle use. */
|
|
op = TREE_OPERAND (stmt, 1);
|
|
vec_oprnd = vect_get_vec_def_for_operand (op, stmt);
|
|
|
|
/* Arguments are ready. create the new vector stmt. */
|
|
*vec_stmt = build2 (MODIFY_EXPR, vectype, vec_dest, vec_oprnd);
|
|
new_temp = make_ssa_name (vec_dest, *vec_stmt);
|
|
TREE_OPERAND (*vec_stmt, 0) = new_temp;
|
|
vect_finish_stmt_generation (stmt, *vec_stmt, bsi);
|
|
|
|
return true;
|
|
}
|
|
|
|
|
|
/* Function vect_min_worthwhile_factor.
|
|
|
|
For a loop where we could vectorize the operation indicated by CODE,
|
|
return the minimum vectorization factor that makes it worthwhile
|
|
to use generic vectors. */
|
|
static int
|
|
vect_min_worthwhile_factor (enum tree_code code)
|
|
{
|
|
switch (code)
|
|
{
|
|
case PLUS_EXPR:
|
|
case MINUS_EXPR:
|
|
case NEGATE_EXPR:
|
|
return 4;
|
|
|
|
case BIT_AND_EXPR:
|
|
case BIT_IOR_EXPR:
|
|
case BIT_XOR_EXPR:
|
|
case BIT_NOT_EXPR:
|
|
return 2;
|
|
|
|
default:
|
|
return INT_MAX;
|
|
}
|
|
}
|
|
|
|
/* Function vectorizable_operation.
|
|
|
|
Check if STMT performs a binary or unary operation that can be vectorized.
|
|
If VEC_STMT is also passed, vectorize the STMT: create a vectorized
|
|
stmt to replace it, put it in VEC_STMT, and insert it at BSI.
|
|
Return FALSE if not a vectorizable STMT, TRUE otherwise. */
|
|
|
|
bool
|
|
vectorizable_operation (tree stmt, block_stmt_iterator *bsi, tree *vec_stmt)
|
|
{
|
|
tree vec_dest;
|
|
tree scalar_dest;
|
|
tree operation;
|
|
tree op0, op1 = NULL;
|
|
tree vec_oprnd0, vec_oprnd1=NULL;
|
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
|
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
|
|
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
|
|
int i;
|
|
enum tree_code code;
|
|
enum machine_mode vec_mode;
|
|
tree new_temp;
|
|
int op_type;
|
|
tree op;
|
|
optab optab;
|
|
|
|
/* Is STMT a vectorizable binary/unary operation? */
|
|
if (TREE_CODE (stmt) != MODIFY_EXPR)
|
|
return false;
|
|
|
|
if (TREE_CODE (TREE_OPERAND (stmt, 0)) != SSA_NAME)
|
|
return false;
|
|
|
|
operation = TREE_OPERAND (stmt, 1);
|
|
code = TREE_CODE (operation);
|
|
optab = optab_for_tree_code (code, vectype);
|
|
|
|
/* Support only unary or binary operations. */
|
|
op_type = TREE_CODE_LENGTH (code);
|
|
if (op_type != unary_op && op_type != binary_op)
|
|
{
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
fprintf (vect_dump, "num. args = %d (not unary/binary op).", op_type);
|
|
return false;
|
|
}
|
|
|
|
for (i = 0; i < op_type; i++)
|
|
{
|
|
op = TREE_OPERAND (operation, i);
|
|
if (!vect_is_simple_use (op, loop_vinfo, NULL))
|
|
{
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
fprintf (vect_dump, "use not simple.");
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/* Supportable by target? */
|
|
if (!optab)
|
|
{
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
fprintf (vect_dump, "no optab.");
|
|
return false;
|
|
}
|
|
vec_mode = TYPE_MODE (vectype);
|
|
if (optab->handlers[(int) vec_mode].insn_code == CODE_FOR_nothing)
|
|
{
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
fprintf (vect_dump, "op not supported by target.");
|
|
if (GET_MODE_SIZE (vec_mode) != UNITS_PER_WORD
|
|
|| LOOP_VINFO_VECT_FACTOR (loop_vinfo)
|
|
< vect_min_worthwhile_factor (code))
|
|
return false;
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
fprintf (vect_dump, "proceeding using word mode.");
|
|
}
|
|
|
|
/* Worthwhile without SIMD support? */
|
|
if (!VECTOR_MODE_P (TYPE_MODE (vectype))
|
|
&& LOOP_VINFO_VECT_FACTOR (loop_vinfo)
|
|
< vect_min_worthwhile_factor (code))
|
|
{
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
fprintf (vect_dump, "not worthwhile without SIMD support.");
|
|
return false;
|
|
}
|
|
|
|
if (!vec_stmt) /* transformation not required. */
|
|
{
|
|
STMT_VINFO_TYPE (stmt_info) = op_vec_info_type;
|
|
return true;
|
|
}
|
|
|
|
/** Transform. **/
|
|
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
fprintf (vect_dump, "transform binary/unary operation.");
|
|
|
|
/* Handle def. */
|
|
scalar_dest = TREE_OPERAND (stmt, 0);
|
|
vec_dest = vect_create_destination_var (scalar_dest, vectype);
|
|
|
|
/* Handle uses. */
|
|
op0 = TREE_OPERAND (operation, 0);
|
|
vec_oprnd0 = vect_get_vec_def_for_operand (op0, stmt);
|
|
|
|
if (op_type == binary_op)
|
|
{
|
|
op1 = TREE_OPERAND (operation, 1);
|
|
vec_oprnd1 = vect_get_vec_def_for_operand (op1, stmt);
|
|
}
|
|
|
|
/* Arguments are ready. create the new vector stmt. */
|
|
|
|
if (op_type == binary_op)
|
|
*vec_stmt = build2 (MODIFY_EXPR, vectype, vec_dest,
|
|
build2 (code, vectype, vec_oprnd0, vec_oprnd1));
|
|
else
|
|
*vec_stmt = build2 (MODIFY_EXPR, vectype, vec_dest,
|
|
build1 (code, vectype, vec_oprnd0));
|
|
new_temp = make_ssa_name (vec_dest, *vec_stmt);
|
|
TREE_OPERAND (*vec_stmt, 0) = new_temp;
|
|
vect_finish_stmt_generation (stmt, *vec_stmt, bsi);
|
|
|
|
return true;
|
|
}
|
|
|
|
|
|
/* Function vectorizable_store.
|
|
|
|
Check if STMT defines a non scalar data-ref (array/pointer/structure) that
|
|
can be vectorized.
|
|
If VEC_STMT is also passed, vectorize the STMT: create a vectorized
|
|
stmt to replace it, put it in VEC_STMT, and insert it at BSI.
|
|
Return FALSE if not a vectorizable STMT, TRUE otherwise. */
|
|
|
|
bool
|
|
vectorizable_store (tree stmt, block_stmt_iterator *bsi, tree *vec_stmt)
|
|
{
|
|
tree scalar_dest;
|
|
tree data_ref;
|
|
tree op;
|
|
tree vec_oprnd1;
|
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
|
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
|
|
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
|
|
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
|
|
enum machine_mode vec_mode;
|
|
tree dummy;
|
|
enum dr_alignment_support alignment_support_cheme;
|
|
ssa_op_iter iter;
|
|
tree def;
|
|
|
|
/* Is vectorizable store? */
|
|
|
|
if (TREE_CODE (stmt) != MODIFY_EXPR)
|
|
return false;
|
|
|
|
scalar_dest = TREE_OPERAND (stmt, 0);
|
|
if (TREE_CODE (scalar_dest) != ARRAY_REF
|
|
&& TREE_CODE (scalar_dest) != INDIRECT_REF)
|
|
return false;
|
|
|
|
op = TREE_OPERAND (stmt, 1);
|
|
if (!vect_is_simple_use (op, loop_vinfo, NULL))
|
|
{
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
fprintf (vect_dump, "use not simple.");
|
|
return false;
|
|
}
|
|
|
|
vec_mode = TYPE_MODE (vectype);
|
|
/* FORNOW. In some cases can vectorize even if data-type not supported
|
|
(e.g. - array initialization with 0). */
|
|
if (mov_optab->handlers[(int)vec_mode].insn_code == CODE_FOR_nothing)
|
|
return false;
|
|
|
|
if (!STMT_VINFO_DATA_REF (stmt_info))
|
|
return false;
|
|
|
|
|
|
if (!vec_stmt) /* transformation not required. */
|
|
{
|
|
STMT_VINFO_TYPE (stmt_info) = store_vec_info_type;
|
|
return true;
|
|
}
|
|
|
|
/** Transform. **/
|
|
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
fprintf (vect_dump, "transform store");
|
|
|
|
alignment_support_cheme = vect_supportable_dr_alignment (dr);
|
|
gcc_assert (alignment_support_cheme);
|
|
gcc_assert (alignment_support_cheme == dr_aligned); /* FORNOW */
|
|
|
|
/* Handle use - get the vectorized def from the defining stmt. */
|
|
vec_oprnd1 = vect_get_vec_def_for_operand (op, stmt);
|
|
|
|
/* Handle def. */
|
|
/* FORNOW: make sure the data reference is aligned. */
|
|
vect_align_data_ref (stmt);
|
|
data_ref = vect_create_data_ref_ptr (stmt, bsi, NULL_TREE, &dummy, false);
|
|
data_ref = build_fold_indirect_ref (data_ref);
|
|
|
|
/* Arguments are ready. create the new vector stmt. */
|
|
*vec_stmt = build2 (MODIFY_EXPR, vectype, data_ref, vec_oprnd1);
|
|
vect_finish_stmt_generation (stmt, *vec_stmt, bsi);
|
|
|
|
/* Mark all non-SSA variables in the statement for rewriting. */
|
|
mark_new_vars_to_rename (*vec_stmt);
|
|
|
|
/* The new vectorized statement will have better aliasing
|
|
information, so some of the virtual definitions of the old
|
|
statement will likely disappear from the IL. Mark them to have
|
|
their SSA form updated. */
|
|
FOR_EACH_SSA_TREE_OPERAND (def, stmt, iter, SSA_OP_VMAYDEF)
|
|
mark_sym_for_renaming (SSA_NAME_VAR (def));
|
|
|
|
return true;
|
|
}
|
|
|
|
|
|
/* vectorizable_load.
|
|
|
|
Check if STMT reads a non scalar data-ref (array/pointer/structure) that
|
|
can be vectorized.
|
|
If VEC_STMT is also passed, vectorize the STMT: create a vectorized
|
|
stmt to replace it, put it in VEC_STMT, and insert it at BSI.
|
|
Return FALSE if not a vectorizable STMT, TRUE otherwise. */
|
|
|
|
bool
|
|
vectorizable_load (tree stmt, block_stmt_iterator *bsi, tree *vec_stmt)
|
|
{
|
|
tree scalar_dest;
|
|
tree vec_dest = NULL;
|
|
tree data_ref = NULL;
|
|
tree op;
|
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
|
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
|
|
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
|
|
tree new_temp;
|
|
int mode;
|
|
tree init_addr;
|
|
tree new_stmt;
|
|
tree dummy;
|
|
basic_block new_bb;
|
|
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
|
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
|
edge pe = loop_preheader_edge (loop);
|
|
enum dr_alignment_support alignment_support_cheme;
|
|
|
|
/* Is vectorizable load? */
|
|
|
|
if (TREE_CODE (stmt) != MODIFY_EXPR)
|
|
return false;
|
|
|
|
scalar_dest = TREE_OPERAND (stmt, 0);
|
|
if (TREE_CODE (scalar_dest) != SSA_NAME)
|
|
return false;
|
|
|
|
op = TREE_OPERAND (stmt, 1);
|
|
if (TREE_CODE (op) != ARRAY_REF && TREE_CODE (op) != INDIRECT_REF)
|
|
return false;
|
|
|
|
if (!STMT_VINFO_DATA_REF (stmt_info))
|
|
return false;
|
|
|
|
mode = (int) TYPE_MODE (vectype);
|
|
|
|
/* FORNOW. In some cases can vectorize even if data-type not supported
|
|
(e.g. - data copies). */
|
|
if (mov_optab->handlers[mode].insn_code == CODE_FOR_nothing)
|
|
{
|
|
if (vect_print_dump_info (REPORT_DETAILS, LOOP_LOC (loop_vinfo)))
|
|
fprintf (vect_dump, "Aligned load, but unsupported type.");
|
|
return false;
|
|
}
|
|
|
|
if (!vec_stmt) /* transformation not required. */
|
|
{
|
|
STMT_VINFO_TYPE (stmt_info) = load_vec_info_type;
|
|
return true;
|
|
}
|
|
|
|
/** Transform. **/
|
|
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
fprintf (vect_dump, "transform load.");
|
|
|
|
alignment_support_cheme = vect_supportable_dr_alignment (dr);
|
|
gcc_assert (alignment_support_cheme);
|
|
|
|
if (alignment_support_cheme == dr_aligned
|
|
|| alignment_support_cheme == dr_unaligned_supported)
|
|
{
|
|
/* Create:
|
|
p = initial_addr;
|
|
indx = 0;
|
|
loop {
|
|
vec_dest = *(p);
|
|
indx = indx + 1;
|
|
}
|
|
*/
|
|
|
|
vec_dest = vect_create_destination_var (scalar_dest, vectype);
|
|
data_ref = vect_create_data_ref_ptr (stmt, bsi, NULL_TREE, &dummy, false);
|
|
if (aligned_access_p (dr))
|
|
data_ref = build_fold_indirect_ref (data_ref);
|
|
else
|
|
{
|
|
int mis = DR_MISALIGNMENT (dr);
|
|
tree tmis = (mis == -1 ? size_zero_node : size_int (mis));
|
|
tmis = size_binop (MULT_EXPR, tmis, size_int(BITS_PER_UNIT));
|
|
data_ref = build2 (MISALIGNED_INDIRECT_REF, vectype, data_ref, tmis);
|
|
}
|
|
new_stmt = build2 (MODIFY_EXPR, vectype, vec_dest, data_ref);
|
|
new_temp = make_ssa_name (vec_dest, new_stmt);
|
|
TREE_OPERAND (new_stmt, 0) = new_temp;
|
|
vect_finish_stmt_generation (stmt, new_stmt, bsi);
|
|
copy_virtual_operands (new_stmt, stmt);
|
|
}
|
|
else if (alignment_support_cheme == dr_unaligned_software_pipeline)
|
|
{
|
|
/* Create:
|
|
p1 = initial_addr;
|
|
msq_init = *(floor(p1))
|
|
p2 = initial_addr + VS - 1;
|
|
magic = have_builtin ? builtin_result : initial_address;
|
|
indx = 0;
|
|
loop {
|
|
p2' = p2 + indx * vectype_size
|
|
lsq = *(floor(p2'))
|
|
vec_dest = realign_load (msq, lsq, magic)
|
|
indx = indx + 1;
|
|
msq = lsq;
|
|
}
|
|
*/
|
|
|
|
tree offset;
|
|
tree magic;
|
|
tree phi_stmt;
|
|
tree msq_init;
|
|
tree msq, lsq;
|
|
tree dataref_ptr;
|
|
tree params;
|
|
|
|
/* <1> Create msq_init = *(floor(p1)) in the loop preheader */
|
|
vec_dest = vect_create_destination_var (scalar_dest, vectype);
|
|
data_ref = vect_create_data_ref_ptr (stmt, bsi, NULL_TREE,
|
|
&init_addr, true);
|
|
data_ref = build1 (ALIGN_INDIRECT_REF, vectype, data_ref);
|
|
new_stmt = build2 (MODIFY_EXPR, vectype, vec_dest, data_ref);
|
|
new_temp = make_ssa_name (vec_dest, new_stmt);
|
|
TREE_OPERAND (new_stmt, 0) = new_temp;
|
|
new_bb = bsi_insert_on_edge_immediate (pe, new_stmt);
|
|
gcc_assert (!new_bb);
|
|
msq_init = TREE_OPERAND (new_stmt, 0);
|
|
copy_virtual_operands (new_stmt, stmt);
|
|
update_vuses_to_preheader (new_stmt, loop);
|
|
|
|
|
|
/* <2> Create lsq = *(floor(p2')) in the loop */
|
|
offset = build_int_cst (integer_type_node,
|
|
TYPE_VECTOR_SUBPARTS (vectype));
|
|
offset = int_const_binop (MINUS_EXPR, offset, integer_one_node, 1);
|
|
vec_dest = vect_create_destination_var (scalar_dest, vectype);
|
|
dataref_ptr = vect_create_data_ref_ptr (stmt, bsi, offset, &dummy, false);
|
|
data_ref = build1 (ALIGN_INDIRECT_REF, vectype, dataref_ptr);
|
|
new_stmt = build2 (MODIFY_EXPR, vectype, vec_dest, data_ref);
|
|
new_temp = make_ssa_name (vec_dest, new_stmt);
|
|
TREE_OPERAND (new_stmt, 0) = new_temp;
|
|
vect_finish_stmt_generation (stmt, new_stmt, bsi);
|
|
lsq = TREE_OPERAND (new_stmt, 0);
|
|
copy_virtual_operands (new_stmt, stmt);
|
|
|
|
|
|
/* <3> */
|
|
if (targetm.vectorize.builtin_mask_for_load)
|
|
{
|
|
/* Create permutation mask, if required, in loop preheader. */
|
|
tree builtin_decl;
|
|
params = build_tree_list (NULL_TREE, init_addr);
|
|
vec_dest = vect_create_destination_var (scalar_dest, vectype);
|
|
builtin_decl = targetm.vectorize.builtin_mask_for_load ();
|
|
new_stmt = build_function_call_expr (builtin_decl, params);
|
|
new_stmt = build2 (MODIFY_EXPR, vectype, vec_dest, new_stmt);
|
|
new_temp = make_ssa_name (vec_dest, new_stmt);
|
|
TREE_OPERAND (new_stmt, 0) = new_temp;
|
|
new_bb = bsi_insert_on_edge_immediate (pe, new_stmt);
|
|
gcc_assert (!new_bb);
|
|
magic = TREE_OPERAND (new_stmt, 0);
|
|
|
|
/* The result of the CALL_EXPR to this builtin is determined from
|
|
the value of the parameter and no global variables are touched
|
|
which makes the builtin a "const" function. Requiring the
|
|
builtin to have the "const" attribute makes it unnecessary
|
|
to call mark_call_clobbered_vars_to_rename. */
|
|
gcc_assert (TREE_READONLY (builtin_decl));
|
|
}
|
|
else
|
|
{
|
|
/* Use current address instead of init_addr for reduced reg pressure.
|
|
*/
|
|
magic = dataref_ptr;
|
|
}
|
|
|
|
|
|
/* <4> Create msq = phi <msq_init, lsq> in loop */
|
|
vec_dest = vect_create_destination_var (scalar_dest, vectype);
|
|
msq = make_ssa_name (vec_dest, NULL_TREE);
|
|
phi_stmt = create_phi_node (msq, loop->header); /* CHECKME */
|
|
SSA_NAME_DEF_STMT (msq) = phi_stmt;
|
|
add_phi_arg (phi_stmt, msq_init, loop_preheader_edge (loop));
|
|
add_phi_arg (phi_stmt, lsq, loop_latch_edge (loop));
|
|
|
|
|
|
/* <5> Create <vec_dest = realign_load (msq, lsq, magic)> in loop */
|
|
vec_dest = vect_create_destination_var (scalar_dest, vectype);
|
|
new_stmt = build3 (REALIGN_LOAD_EXPR, vectype, msq, lsq, magic);
|
|
new_stmt = build2 (MODIFY_EXPR, vectype, vec_dest, new_stmt);
|
|
new_temp = make_ssa_name (vec_dest, new_stmt);
|
|
TREE_OPERAND (new_stmt, 0) = new_temp;
|
|
vect_finish_stmt_generation (stmt, new_stmt, bsi);
|
|
}
|
|
else
|
|
gcc_unreachable ();
|
|
|
|
*vec_stmt = new_stmt;
|
|
return true;
|
|
}
|
|
|
|
/* Function vect_is_simple_cond.
|
|
|
|
Input:
|
|
LOOP - the loop that is being vectorized.
|
|
COND - Condition that is checked for simple use.
|
|
|
|
Returns whether a COND can be vectorized. Checks whether
|
|
condition operands are supportable using vec_is_simple_use. */
|
|
|
|
static bool
|
|
vect_is_simple_cond (tree cond, loop_vec_info loop_vinfo)
|
|
{
|
|
tree lhs, rhs;
|
|
|
|
if (!COMPARISON_CLASS_P (cond))
|
|
return false;
|
|
|
|
lhs = TREE_OPERAND (cond, 0);
|
|
rhs = TREE_OPERAND (cond, 1);
|
|
|
|
if (TREE_CODE (lhs) == SSA_NAME)
|
|
{
|
|
tree lhs_def_stmt = SSA_NAME_DEF_STMT (lhs);
|
|
if (!vect_is_simple_use (lhs, loop_vinfo, &lhs_def_stmt))
|
|
return false;
|
|
}
|
|
else if (TREE_CODE (lhs) != INTEGER_CST && TREE_CODE (lhs) != REAL_CST)
|
|
return false;
|
|
|
|
if (TREE_CODE (rhs) == SSA_NAME)
|
|
{
|
|
tree rhs_def_stmt = SSA_NAME_DEF_STMT (rhs);
|
|
if (!vect_is_simple_use (rhs, loop_vinfo, &rhs_def_stmt))
|
|
return false;
|
|
}
|
|
else if (TREE_CODE (rhs) != INTEGER_CST && TREE_CODE (rhs) != REAL_CST)
|
|
return false;
|
|
|
|
return true;
|
|
}
|
|
|
|
/* vectorizable_condition.
|
|
|
|
Check if STMT is conditional modify expression that can be vectorized.
|
|
If VEC_STMT is also passed, vectorize the STMT: create a vectorized
|
|
stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it
|
|
at BSI.
|
|
|
|
Return FALSE if not a vectorizable STMT, TRUE otherwise. */
|
|
|
|
bool
|
|
vectorizable_condition (tree stmt, block_stmt_iterator *bsi, tree *vec_stmt)
|
|
{
|
|
tree scalar_dest = NULL_TREE;
|
|
tree vec_dest = NULL_TREE;
|
|
tree op = NULL_TREE;
|
|
tree cond_expr, then_clause, else_clause;
|
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
|
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
|
|
tree vec_cond_lhs, vec_cond_rhs, vec_then_clause, vec_else_clause;
|
|
tree vec_compare, vec_cond_expr;
|
|
tree new_temp;
|
|
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
|
|
enum machine_mode vec_mode;
|
|
|
|
if (!STMT_VINFO_RELEVANT_P (stmt_info))
|
|
return false;
|
|
|
|
if (TREE_CODE (stmt) != MODIFY_EXPR)
|
|
return false;
|
|
|
|
op = TREE_OPERAND (stmt, 1);
|
|
|
|
if (TREE_CODE (op) != COND_EXPR)
|
|
return false;
|
|
|
|
cond_expr = TREE_OPERAND (op, 0);
|
|
then_clause = TREE_OPERAND (op, 1);
|
|
else_clause = TREE_OPERAND (op, 2);
|
|
|
|
if (!vect_is_simple_cond (cond_expr, loop_vinfo))
|
|
return false;
|
|
|
|
if (TREE_CODE (then_clause) == SSA_NAME)
|
|
{
|
|
tree then_def_stmt = SSA_NAME_DEF_STMT (then_clause);
|
|
if (!vect_is_simple_use (then_clause, loop_vinfo, &then_def_stmt))
|
|
return false;
|
|
}
|
|
else if (TREE_CODE (then_clause) != INTEGER_CST
|
|
&& TREE_CODE (then_clause) != REAL_CST)
|
|
return false;
|
|
|
|
if (TREE_CODE (else_clause) == SSA_NAME)
|
|
{
|
|
tree else_def_stmt = SSA_NAME_DEF_STMT (else_clause);
|
|
if (!vect_is_simple_use (else_clause, loop_vinfo, &else_def_stmt))
|
|
return false;
|
|
}
|
|
else if (TREE_CODE (else_clause) != INTEGER_CST
|
|
&& TREE_CODE (else_clause) != REAL_CST)
|
|
return false;
|
|
|
|
|
|
vec_mode = TYPE_MODE (vectype);
|
|
|
|
if (!vec_stmt)
|
|
{
|
|
STMT_VINFO_TYPE (stmt_info) = condition_vec_info_type;
|
|
return expand_vec_cond_expr_p (op, vec_mode);
|
|
}
|
|
|
|
/* Transform */
|
|
|
|
/* Handle def. */
|
|
scalar_dest = TREE_OPERAND (stmt, 0);
|
|
vec_dest = vect_create_destination_var (scalar_dest, vectype);
|
|
|
|
/* Handle cond expr. */
|
|
vec_cond_lhs =
|
|
vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0), stmt);
|
|
vec_cond_rhs =
|
|
vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1), stmt);
|
|
vec_then_clause = vect_get_vec_def_for_operand (then_clause, stmt);
|
|
vec_else_clause = vect_get_vec_def_for_operand (else_clause, stmt);
|
|
|
|
/* Arguments are ready. create the new vector stmt. */
|
|
vec_compare = build2 (TREE_CODE (cond_expr), vectype,
|
|
vec_cond_lhs, vec_cond_rhs);
|
|
vec_cond_expr = build (VEC_COND_EXPR, vectype,
|
|
vec_compare, vec_then_clause, vec_else_clause);
|
|
|
|
*vec_stmt = build2 (MODIFY_EXPR, vectype, vec_dest, vec_cond_expr);
|
|
new_temp = make_ssa_name (vec_dest, *vec_stmt);
|
|
TREE_OPERAND (*vec_stmt, 0) = new_temp;
|
|
vect_finish_stmt_generation (stmt, *vec_stmt, bsi);
|
|
|
|
return true;
|
|
}
|
|
|
|
/* Function vect_transform_stmt.
|
|
|
|
Create a vectorized stmt to replace STMT, and insert it at BSI. */
|
|
|
|
bool
|
|
vect_transform_stmt (tree stmt, block_stmt_iterator *bsi)
|
|
{
|
|
bool is_store = false;
|
|
tree vec_stmt = NULL_TREE;
|
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
|
bool done;
|
|
|
|
switch (STMT_VINFO_TYPE (stmt_info))
|
|
{
|
|
case op_vec_info_type:
|
|
done = vectorizable_operation (stmt, bsi, &vec_stmt);
|
|
gcc_assert (done);
|
|
break;
|
|
|
|
case assignment_vec_info_type:
|
|
done = vectorizable_assignment (stmt, bsi, &vec_stmt);
|
|
gcc_assert (done);
|
|
break;
|
|
|
|
case load_vec_info_type:
|
|
done = vectorizable_load (stmt, bsi, &vec_stmt);
|
|
gcc_assert (done);
|
|
break;
|
|
|
|
case store_vec_info_type:
|
|
done = vectorizable_store (stmt, bsi, &vec_stmt);
|
|
gcc_assert (done);
|
|
is_store = true;
|
|
break;
|
|
|
|
case condition_vec_info_type:
|
|
done = vectorizable_condition (stmt, bsi, &vec_stmt);
|
|
gcc_assert (done);
|
|
break;
|
|
|
|
default:
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
fprintf (vect_dump, "stmt not supported.");
|
|
gcc_unreachable ();
|
|
}
|
|
|
|
STMT_VINFO_VEC_STMT (stmt_info) = vec_stmt;
|
|
|
|
return is_store;
|
|
}
|
|
|
|
|
|
/* This function builds ni_name = number of iterations loop executes
|
|
on the loop preheader. */
|
|
|
|
static tree
|
|
vect_build_loop_niters (loop_vec_info loop_vinfo)
|
|
{
|
|
tree ni_name, stmt, var;
|
|
edge pe;
|
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
|
tree ni = unshare_expr (LOOP_VINFO_NITERS (loop_vinfo));
|
|
|
|
var = create_tmp_var (TREE_TYPE (ni), "niters");
|
|
add_referenced_tmp_var (var);
|
|
ni_name = force_gimple_operand (ni, &stmt, false, var);
|
|
|
|
pe = loop_preheader_edge (loop);
|
|
if (stmt)
|
|
{
|
|
basic_block new_bb = bsi_insert_on_edge_immediate (pe, stmt);
|
|
gcc_assert (!new_bb);
|
|
}
|
|
|
|
return ni_name;
|
|
}
|
|
|
|
|
|
/* This function generates the following statements:
|
|
|
|
ni_name = number of iterations loop executes
|
|
ratio = ni_name / vf
|
|
ratio_mult_vf_name = ratio * vf
|
|
|
|
and places them at the loop preheader edge. */
|
|
|
|
static void
|
|
vect_generate_tmps_on_preheader (loop_vec_info loop_vinfo,
|
|
tree *ni_name_ptr,
|
|
tree *ratio_mult_vf_name_ptr,
|
|
tree *ratio_name_ptr)
|
|
{
|
|
|
|
edge pe;
|
|
basic_block new_bb;
|
|
tree stmt, ni_name;
|
|
tree var;
|
|
tree ratio_name;
|
|
tree ratio_mult_vf_name;
|
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
|
tree ni = LOOP_VINFO_NITERS (loop_vinfo);
|
|
int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
|
|
tree log_vf = build_int_cst (unsigned_type_node, exact_log2 (vf));
|
|
|
|
pe = loop_preheader_edge (loop);
|
|
|
|
/* Generate temporary variable that contains
|
|
number of iterations loop executes. */
|
|
|
|
ni_name = vect_build_loop_niters (loop_vinfo);
|
|
|
|
/* Create: ratio = ni >> log2(vf) */
|
|
|
|
var = create_tmp_var (TREE_TYPE (ni), "bnd");
|
|
add_referenced_tmp_var (var);
|
|
ratio_name = make_ssa_name (var, NULL_TREE);
|
|
stmt = build2 (MODIFY_EXPR, void_type_node, ratio_name,
|
|
build2 (RSHIFT_EXPR, TREE_TYPE (ni_name), ni_name, log_vf));
|
|
SSA_NAME_DEF_STMT (ratio_name) = stmt;
|
|
|
|
pe = loop_preheader_edge (loop);
|
|
new_bb = bsi_insert_on_edge_immediate (pe, stmt);
|
|
gcc_assert (!new_bb);
|
|
|
|
/* Create: ratio_mult_vf = ratio << log2 (vf). */
|
|
|
|
var = create_tmp_var (TREE_TYPE (ni), "ratio_mult_vf");
|
|
add_referenced_tmp_var (var);
|
|
ratio_mult_vf_name = make_ssa_name (var, NULL_TREE);
|
|
stmt = build2 (MODIFY_EXPR, void_type_node, ratio_mult_vf_name,
|
|
build2 (LSHIFT_EXPR, TREE_TYPE (ratio_name), ratio_name, log_vf));
|
|
SSA_NAME_DEF_STMT (ratio_mult_vf_name) = stmt;
|
|
|
|
pe = loop_preheader_edge (loop);
|
|
new_bb = bsi_insert_on_edge_immediate (pe, stmt);
|
|
gcc_assert (!new_bb);
|
|
|
|
*ni_name_ptr = ni_name;
|
|
*ratio_mult_vf_name_ptr = ratio_mult_vf_name;
|
|
*ratio_name_ptr = ratio_name;
|
|
|
|
return;
|
|
}
|
|
|
|
|
|
/* Function update_vuses_to_preheader.
|
|
|
|
Input:
|
|
STMT - a statement with potential VUSEs.
|
|
LOOP - the loop whose preheader will contain STMT.
|
|
|
|
It's possible to vectorize a loop even though an SSA_NAME from a VUSE
|
|
appears to be defined in a V_MAY_DEF in another statement in a loop.
|
|
One such case is when the VUSE is at the dereference of a __restricted__
|
|
pointer in a load and the V_MAY_DEF is at the dereference of a different
|
|
__restricted__ pointer in a store. Vectorization may result in
|
|
copy_virtual_uses being called to copy the problematic VUSE to a new
|
|
statement that is being inserted in the loop preheader. This procedure
|
|
is called to change the SSA_NAME in the new statement's VUSE from the
|
|
SSA_NAME updated in the loop to the related SSA_NAME available on the
|
|
path entering the loop.
|
|
|
|
When this function is called, we have the following situation:
|
|
|
|
# vuse <name1>
|
|
S1: vload
|
|
do {
|
|
# name1 = phi < name0 , name2>
|
|
|
|
# vuse <name1>
|
|
S2: vload
|
|
|
|
# name2 = vdef <name1>
|
|
S3: vstore
|
|
|
|
}while...
|
|
|
|
Stmt S1 was created in the loop preheader block as part of misaligned-load
|
|
handling. This function fixes the name of the vuse of S1 from 'name1' to
|
|
'name0'. */
|
|
|
|
static void
|
|
update_vuses_to_preheader (tree stmt, struct loop *loop)
|
|
{
|
|
basic_block header_bb = loop->header;
|
|
edge preheader_e = loop_preheader_edge (loop);
|
|
ssa_op_iter iter;
|
|
use_operand_p use_p;
|
|
|
|
FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_VUSE)
|
|
{
|
|
tree ssa_name = USE_FROM_PTR (use_p);
|
|
tree def_stmt = SSA_NAME_DEF_STMT (ssa_name);
|
|
tree name_var = SSA_NAME_VAR (ssa_name);
|
|
basic_block bb = bb_for_stmt (def_stmt);
|
|
|
|
/* For a use before any definitions, def_stmt is a NOP_EXPR. */
|
|
if (!IS_EMPTY_STMT (def_stmt)
|
|
&& flow_bb_inside_loop_p (loop, bb))
|
|
{
|
|
/* If the block containing the statement defining the SSA_NAME
|
|
is in the loop then it's necessary to find the definition
|
|
outside the loop using the PHI nodes of the header. */
|
|
tree phi;
|
|
bool updated = false;
|
|
|
|
for (phi = phi_nodes (header_bb); phi; phi = TREE_CHAIN (phi))
|
|
{
|
|
if (SSA_NAME_VAR (PHI_RESULT (phi)) == name_var)
|
|
{
|
|
SET_USE (use_p, PHI_ARG_DEF (phi, preheader_e->dest_idx));
|
|
updated = true;
|
|
break;
|
|
}
|
|
}
|
|
gcc_assert (updated);
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
/* Function vect_update_ivs_after_vectorizer.
|
|
|
|
"Advance" the induction variables of LOOP to the value they should take
|
|
after the execution of LOOP. This is currently necessary because the
|
|
vectorizer does not handle induction variables that are used after the
|
|
loop. Such a situation occurs when the last iterations of LOOP are
|
|
peeled, because:
|
|
1. We introduced new uses after LOOP for IVs that were not originally used
|
|
after LOOP: the IVs of LOOP are now used by an epilog loop.
|
|
2. LOOP is going to be vectorized; this means that it will iterate N/VF
|
|
times, whereas the loop IVs should be bumped N times.
|
|
|
|
Input:
|
|
- LOOP - a loop that is going to be vectorized. The last few iterations
|
|
of LOOP were peeled.
|
|
- NITERS - the number of iterations that LOOP executes (before it is
|
|
vectorized). i.e, the number of times the ivs should be bumped.
|
|
- UPDATE_E - a successor edge of LOOP->exit that is on the (only) path
|
|
coming out from LOOP on which there are uses of the LOOP ivs
|
|
(this is the path from LOOP->exit to epilog_loop->preheader).
|
|
|
|
The new definitions of the ivs are placed in LOOP->exit.
|
|
The phi args associated with the edge UPDATE_E in the bb
|
|
UPDATE_E->dest are updated accordingly.
|
|
|
|
Assumption 1: Like the rest of the vectorizer, this function assumes
|
|
a single loop exit that has a single predecessor.
|
|
|
|
Assumption 2: The phi nodes in the LOOP header and in update_bb are
|
|
organized in the same order.
|
|
|
|
Assumption 3: The access function of the ivs is simple enough (see
|
|
vect_can_advance_ivs_p). This assumption will be relaxed in the future.
|
|
|
|
Assumption 4: Exactly one of the successors of LOOP exit-bb is on a path
|
|
coming out of LOOP on which the ivs of LOOP are used (this is the path
|
|
that leads to the epilog loop; other paths skip the epilog loop). This
|
|
path starts with the edge UPDATE_E, and its destination (denoted update_bb)
|
|
needs to have its phis updated.
|
|
*/
|
|
|
|
static void
|
|
vect_update_ivs_after_vectorizer (loop_vec_info loop_vinfo, tree niters,
|
|
edge update_e)
|
|
{
|
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
|
basic_block exit_bb = loop->single_exit->dest;
|
|
tree phi, phi1;
|
|
basic_block update_bb = update_e->dest;
|
|
|
|
/* gcc_assert (vect_can_advance_ivs_p (loop_vinfo)); */
|
|
|
|
/* Make sure there exists a single-predecessor exit bb: */
|
|
gcc_assert (single_pred_p (exit_bb));
|
|
|
|
for (phi = phi_nodes (loop->header), phi1 = phi_nodes (update_bb);
|
|
phi && phi1;
|
|
phi = PHI_CHAIN (phi), phi1 = PHI_CHAIN (phi1))
|
|
{
|
|
tree access_fn = NULL;
|
|
tree evolution_part;
|
|
tree init_expr;
|
|
tree step_expr;
|
|
tree var, stmt, ni, ni_name;
|
|
block_stmt_iterator last_bsi;
|
|
|
|
/* Skip virtual phi's. */
|
|
if (!is_gimple_reg (SSA_NAME_VAR (PHI_RESULT (phi))))
|
|
{
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
fprintf (vect_dump, "virtual phi. skip.");
|
|
continue;
|
|
}
|
|
|
|
access_fn = analyze_scalar_evolution (loop, PHI_RESULT (phi));
|
|
gcc_assert (access_fn);
|
|
evolution_part =
|
|
unshare_expr (evolution_part_in_loop_num (access_fn, loop->num));
|
|
gcc_assert (evolution_part != NULL_TREE);
|
|
|
|
/* FORNOW: We do not support IVs whose evolution function is a polynomial
|
|
of degree >= 2 or exponential. */
|
|
gcc_assert (!tree_is_chrec (evolution_part));
|
|
|
|
step_expr = evolution_part;
|
|
init_expr = unshare_expr (initial_condition_in_loop_num (access_fn,
|
|
loop->num));
|
|
|
|
ni = build2 (PLUS_EXPR, TREE_TYPE (init_expr),
|
|
build2 (MULT_EXPR, TREE_TYPE (niters),
|
|
niters, step_expr), init_expr);
|
|
|
|
var = create_tmp_var (TREE_TYPE (init_expr), "tmp");
|
|
add_referenced_tmp_var (var);
|
|
|
|
ni_name = force_gimple_operand (ni, &stmt, false, var);
|
|
|
|
/* Insert stmt into exit_bb. */
|
|
last_bsi = bsi_last (exit_bb);
|
|
if (stmt)
|
|
bsi_insert_before (&last_bsi, stmt, BSI_SAME_STMT);
|
|
|
|
/* Fix phi expressions in the successor bb. */
|
|
SET_PHI_ARG_DEF (phi1, update_e->dest_idx, ni_name);
|
|
}
|
|
}
|
|
|
|
|
|
/* Function vect_do_peeling_for_loop_bound
|
|
|
|
Peel the last iterations of the loop represented by LOOP_VINFO.
|
|
The peeled iterations form a new epilog loop. Given that the loop now
|
|
iterates NITERS times, the new epilog loop iterates
|
|
NITERS % VECTORIZATION_FACTOR times.
|
|
|
|
The original loop will later be made to iterate
|
|
NITERS / VECTORIZATION_FACTOR times (this value is placed into RATIO). */
|
|
|
|
static void
|
|
vect_do_peeling_for_loop_bound (loop_vec_info loop_vinfo, tree *ratio,
|
|
struct loops *loops)
|
|
{
|
|
|
|
tree ni_name, ratio_mult_vf_name;
|
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
|
struct loop *new_loop;
|
|
edge update_e;
|
|
basic_block preheader;
|
|
#ifdef ENABLE_CHECKING
|
|
int loop_num;
|
|
#endif
|
|
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
fprintf (vect_dump, "=== vect_transtorm_for_unknown_loop_bound ===");
|
|
|
|
/* Generate the following variables on the preheader of original loop:
|
|
|
|
ni_name = number of iteration the original loop executes
|
|
ratio = ni_name / vf
|
|
ratio_mult_vf_name = ratio * vf */
|
|
vect_generate_tmps_on_preheader (loop_vinfo, &ni_name,
|
|
&ratio_mult_vf_name, ratio);
|
|
|
|
#ifdef ENABLE_CHECKING
|
|
loop_num = loop->num;
|
|
#endif
|
|
new_loop = slpeel_tree_peel_loop_to_edge (loop, loops, loop->single_exit,
|
|
ratio_mult_vf_name, ni_name, false);
|
|
#ifdef ENABLE_CHECKING
|
|
gcc_assert (new_loop);
|
|
gcc_assert (loop_num == loop->num);
|
|
slpeel_verify_cfg_after_peeling (loop, new_loop);
|
|
#endif
|
|
|
|
/* A guard that controls whether the new_loop is to be executed or skipped
|
|
is placed in LOOP->exit. LOOP->exit therefore has two successors - one
|
|
is the preheader of NEW_LOOP, where the IVs from LOOP are used. The other
|
|
is a bb after NEW_LOOP, where these IVs are not used. Find the edge that
|
|
is on the path where the LOOP IVs are used and need to be updated. */
|
|
|
|
preheader = loop_preheader_edge (new_loop)->src;
|
|
if (EDGE_PRED (preheader, 0)->src == loop->single_exit->dest)
|
|
update_e = EDGE_PRED (preheader, 0);
|
|
else
|
|
update_e = EDGE_PRED (preheader, 1);
|
|
|
|
/* Update IVs of original loop as if they were advanced
|
|
by ratio_mult_vf_name steps. */
|
|
vect_update_ivs_after_vectorizer (loop_vinfo, ratio_mult_vf_name, update_e);
|
|
|
|
/* After peeling we have to reset scalar evolution analyzer. */
|
|
scev_reset ();
|
|
|
|
return;
|
|
}
|
|
|
|
|
|
/* Function vect_gen_niters_for_prolog_loop
|
|
|
|
Set the number of iterations for the loop represented by LOOP_VINFO
|
|
to the minimum between LOOP_NITERS (the original iteration count of the loop)
|
|
and the misalignment of DR - the data reference recorded in
|
|
LOOP_VINFO_UNALIGNED_DR (LOOP_VINFO). As a result, after the execution of
|
|
this loop, the data reference DR will refer to an aligned location.
|
|
|
|
The following computation is generated:
|
|
|
|
If the misalignment of DR is known at compile time:
|
|
addr_mis = int mis = DR_MISALIGNMENT (dr);
|
|
Else, compute address misalignment in bytes:
|
|
addr_mis = addr & (vectype_size - 1)
|
|
|
|
prolog_niters = min ( LOOP_NITERS , (VF - addr_mis/elem_size)&(VF-1) )
|
|
|
|
(elem_size = element type size; an element is the scalar element
|
|
whose type is the inner type of the vectype) */
|
|
|
|
static tree
|
|
vect_gen_niters_for_prolog_loop (loop_vec_info loop_vinfo, tree loop_niters)
|
|
{
|
|
struct data_reference *dr = LOOP_VINFO_UNALIGNED_DR (loop_vinfo);
|
|
int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
|
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
|
tree var, stmt;
|
|
tree iters, iters_name;
|
|
edge pe;
|
|
basic_block new_bb;
|
|
tree dr_stmt = DR_STMT (dr);
|
|
stmt_vec_info stmt_info = vinfo_for_stmt (dr_stmt);
|
|
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
|
|
int vectype_align = TYPE_ALIGN (vectype) / BITS_PER_UNIT;
|
|
tree vf_minus_1 = build_int_cst (unsigned_type_node, vf - 1);
|
|
tree niters_type = TREE_TYPE (loop_niters);
|
|
|
|
pe = loop_preheader_edge (loop);
|
|
|
|
if (LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo) > 0)
|
|
{
|
|
int byte_misalign = LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo);
|
|
int element_size = vectype_align/vf;
|
|
int elem_misalign = byte_misalign / element_size;
|
|
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
fprintf (vect_dump, "known alignment = %d.", byte_misalign);
|
|
iters = build_int_cst (niters_type, (vf - elem_misalign)&(vf-1));
|
|
}
|
|
else
|
|
{
|
|
tree new_stmts = NULL_TREE;
|
|
tree start_addr =
|
|
vect_create_addr_base_for_vector_ref (dr_stmt, &new_stmts, NULL_TREE);
|
|
tree ptr_type = TREE_TYPE (start_addr);
|
|
tree size = TYPE_SIZE (ptr_type);
|
|
tree type = lang_hooks.types.type_for_size (tree_low_cst (size, 1), 1);
|
|
tree vectype_size_minus_1 = build_int_cst (type, vectype_align - 1);
|
|
tree elem_size_log =
|
|
build_int_cst (unsigned_type_node, exact_log2 (vectype_align/vf));
|
|
tree vf_tree = build_int_cst (unsigned_type_node, vf);
|
|
tree byte_misalign;
|
|
tree elem_misalign;
|
|
|
|
new_bb = bsi_insert_on_edge_immediate (pe, new_stmts);
|
|
gcc_assert (!new_bb);
|
|
|
|
/* Create: byte_misalign = addr & (vectype_size - 1) */
|
|
byte_misalign =
|
|
build2 (BIT_AND_EXPR, type, start_addr, vectype_size_minus_1);
|
|
|
|
/* Create: elem_misalign = byte_misalign / element_size */
|
|
elem_misalign =
|
|
build2 (RSHIFT_EXPR, unsigned_type_node, byte_misalign, elem_size_log);
|
|
|
|
/* Create: (niters_type) (VF - elem_misalign)&(VF - 1) */
|
|
iters = build2 (MINUS_EXPR, unsigned_type_node, vf_tree, elem_misalign);
|
|
iters = build2 (BIT_AND_EXPR, unsigned_type_node, iters, vf_minus_1);
|
|
iters = fold_convert (niters_type, iters);
|
|
}
|
|
|
|
/* Create: prolog_loop_niters = min (iters, loop_niters) */
|
|
/* If the loop bound is known at compile time we already verified that it is
|
|
greater than vf; since the misalignment ('iters') is at most vf, there's
|
|
no need to generate the MIN_EXPR in this case. */
|
|
if (TREE_CODE (loop_niters) != INTEGER_CST)
|
|
iters = build2 (MIN_EXPR, niters_type, iters, loop_niters);
|
|
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
{
|
|
fprintf (vect_dump, "niters for prolog loop: ");
|
|
print_generic_expr (vect_dump, iters, TDF_SLIM);
|
|
}
|
|
|
|
var = create_tmp_var (niters_type, "prolog_loop_niters");
|
|
add_referenced_tmp_var (var);
|
|
iters_name = force_gimple_operand (iters, &stmt, false, var);
|
|
|
|
/* Insert stmt on loop preheader edge. */
|
|
if (stmt)
|
|
{
|
|
basic_block new_bb = bsi_insert_on_edge_immediate (pe, stmt);
|
|
gcc_assert (!new_bb);
|
|
}
|
|
|
|
return iters_name;
|
|
}
|
|
|
|
|
|
/* Function vect_update_init_of_dr
|
|
|
|
NITERS iterations were peeled from LOOP. DR represents a data reference
|
|
in LOOP. This function updates the information recorded in DR to
|
|
account for the fact that the first NITERS iterations had already been
|
|
executed. Specifically, it updates the OFFSET field of stmt_info. */
|
|
|
|
static void
|
|
vect_update_init_of_dr (struct data_reference *dr, tree niters)
|
|
{
|
|
stmt_vec_info stmt_info = vinfo_for_stmt (DR_STMT (dr));
|
|
tree offset = STMT_VINFO_VECT_INIT_OFFSET (stmt_info);
|
|
|
|
niters = fold (build2 (MULT_EXPR, TREE_TYPE (niters), niters,
|
|
STMT_VINFO_VECT_STEP (stmt_info)));
|
|
offset = fold (build2 (PLUS_EXPR, TREE_TYPE (offset), offset, niters));
|
|
STMT_VINFO_VECT_INIT_OFFSET (stmt_info) = offset;
|
|
}
|
|
|
|
|
|
/* Function vect_update_inits_of_drs
|
|
|
|
NITERS iterations were peeled from the loop represented by LOOP_VINFO.
|
|
This function updates the information recorded for the data references in
|
|
the loop to account for the fact that the first NITERS iterations had
|
|
already been executed. Specifically, it updates the initial_condition of the
|
|
access_function of all the data_references in the loop. */
|
|
|
|
static void
|
|
vect_update_inits_of_drs (loop_vec_info loop_vinfo, tree niters)
|
|
{
|
|
unsigned int i;
|
|
varray_type loop_write_datarefs = LOOP_VINFO_DATAREF_WRITES (loop_vinfo);
|
|
varray_type loop_read_datarefs = LOOP_VINFO_DATAREF_READS (loop_vinfo);
|
|
|
|
if (vect_dump && (dump_flags & TDF_DETAILS))
|
|
fprintf (vect_dump, "=== vect_update_inits_of_dr ===");
|
|
|
|
for (i = 0; i < VARRAY_ACTIVE_SIZE (loop_write_datarefs); i++)
|
|
{
|
|
struct data_reference *dr = VARRAY_GENERIC_PTR (loop_write_datarefs, i);
|
|
vect_update_init_of_dr (dr, niters);
|
|
}
|
|
|
|
for (i = 0; i < VARRAY_ACTIVE_SIZE (loop_read_datarefs); i++)
|
|
{
|
|
struct data_reference *dr = VARRAY_GENERIC_PTR (loop_read_datarefs, i);
|
|
vect_update_init_of_dr (dr, niters);
|
|
}
|
|
}
|
|
|
|
|
|
/* Function vect_do_peeling_for_alignment
|
|
|
|
Peel the first 'niters' iterations of the loop represented by LOOP_VINFO.
|
|
'niters' is set to the misalignment of one of the data references in the
|
|
loop, thereby forcing it to refer to an aligned location at the beginning
|
|
of the execution of this loop. The data reference for which we are
|
|
peeling is recorded in LOOP_VINFO_UNALIGNED_DR. */
|
|
|
|
static void
|
|
vect_do_peeling_for_alignment (loop_vec_info loop_vinfo, struct loops *loops)
|
|
{
|
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
|
tree niters_of_prolog_loop, ni_name;
|
|
tree n_iters;
|
|
struct loop *new_loop;
|
|
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
fprintf (vect_dump, "=== vect_do_peeling_for_alignment ===");
|
|
|
|
ni_name = vect_build_loop_niters (loop_vinfo);
|
|
niters_of_prolog_loop = vect_gen_niters_for_prolog_loop (loop_vinfo, ni_name);
|
|
|
|
/* Peel the prolog loop and iterate it niters_of_prolog_loop. */
|
|
new_loop =
|
|
slpeel_tree_peel_loop_to_edge (loop, loops, loop_preheader_edge (loop),
|
|
niters_of_prolog_loop, ni_name, true);
|
|
#ifdef ENABLE_CHECKING
|
|
gcc_assert (new_loop);
|
|
slpeel_verify_cfg_after_peeling (new_loop, loop);
|
|
#endif
|
|
|
|
/* Update number of times loop executes. */
|
|
n_iters = LOOP_VINFO_NITERS (loop_vinfo);
|
|
LOOP_VINFO_NITERS (loop_vinfo) = fold (build2 (MINUS_EXPR,
|
|
TREE_TYPE (n_iters), n_iters, niters_of_prolog_loop));
|
|
|
|
/* Update the init conditions of the access functions of all data refs. */
|
|
vect_update_inits_of_drs (loop_vinfo, niters_of_prolog_loop);
|
|
|
|
/* After peeling we have to reset scalar evolution analyzer. */
|
|
scev_reset ();
|
|
|
|
return;
|
|
}
|
|
|
|
|
|
/* Function vect_transform_loop.
|
|
|
|
The analysis phase has determined that the loop is vectorizable.
|
|
Vectorize the loop - created vectorized stmts to replace the scalar
|
|
stmts in the loop, and update the loop exit condition. */
|
|
|
|
void
|
|
vect_transform_loop (loop_vec_info loop_vinfo,
|
|
struct loops *loops ATTRIBUTE_UNUSED)
|
|
{
|
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
|
basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
|
|
int nbbs = loop->num_nodes;
|
|
block_stmt_iterator si;
|
|
int i;
|
|
tree ratio = NULL;
|
|
int vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
|
|
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
fprintf (vect_dump, "=== vec_transform_loop ===");
|
|
|
|
|
|
/* Peel the loop if there are data refs with unknown alignment.
|
|
Only one data ref with unknown store is allowed. */
|
|
|
|
if (LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo))
|
|
vect_do_peeling_for_alignment (loop_vinfo, loops);
|
|
|
|
/* If the loop has a symbolic number of iterations 'n' (i.e. it's not a
|
|
compile time constant), or it is a constant that doesn't divide by the
|
|
vectorization factor, then an epilog loop needs to be created.
|
|
We therefore duplicate the loop: the original loop will be vectorized,
|
|
and will compute the first (n/VF) iterations. The second copy of the loop
|
|
will remain scalar and will compute the remaining (n%VF) iterations.
|
|
(VF is the vectorization factor). */
|
|
|
|
if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
|
|
|| (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
|
|
&& LOOP_VINFO_INT_NITERS (loop_vinfo) % vectorization_factor != 0))
|
|
vect_do_peeling_for_loop_bound (loop_vinfo, &ratio, loops);
|
|
else
|
|
ratio = build_int_cst (TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo)),
|
|
LOOP_VINFO_INT_NITERS (loop_vinfo) / vectorization_factor);
|
|
|
|
/* 1) Make sure the loop header has exactly two entries
|
|
2) Make sure we have a preheader basic block. */
|
|
|
|
gcc_assert (EDGE_COUNT (loop->header->preds) == 2);
|
|
|
|
loop_split_edge_with (loop_preheader_edge (loop), NULL);
|
|
|
|
|
|
/* FORNOW: the vectorizer supports only loops which body consist
|
|
of one basic block (header + empty latch). When the vectorizer will
|
|
support more involved loop forms, the order by which the BBs are
|
|
traversed need to be reconsidered. */
|
|
|
|
for (i = 0; i < nbbs; i++)
|
|
{
|
|
basic_block bb = bbs[i];
|
|
|
|
for (si = bsi_start (bb); !bsi_end_p (si);)
|
|
{
|
|
tree stmt = bsi_stmt (si);
|
|
stmt_vec_info stmt_info;
|
|
bool is_store;
|
|
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
{
|
|
fprintf (vect_dump, "------>vectorizing statement: ");
|
|
print_generic_expr (vect_dump, stmt, TDF_SLIM);
|
|
}
|
|
stmt_info = vinfo_for_stmt (stmt);
|
|
gcc_assert (stmt_info);
|
|
if (!STMT_VINFO_RELEVANT_P (stmt_info))
|
|
{
|
|
bsi_next (&si);
|
|
continue;
|
|
}
|
|
#ifdef ENABLE_CHECKING
|
|
/* FORNOW: Verify that all stmts operate on the same number of
|
|
units and no inner unrolling is necessary. */
|
|
gcc_assert
|
|
(TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info))
|
|
== vectorization_factor);
|
|
#endif
|
|
/* -------- vectorize statement ------------ */
|
|
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
|
|
fprintf (vect_dump, "transform statement.");
|
|
|
|
is_store = vect_transform_stmt (stmt, &si);
|
|
if (is_store)
|
|
{
|
|
/* Free the attached stmt_vec_info and remove the stmt. */
|
|
stmt_ann_t ann = stmt_ann (stmt);
|
|
free (stmt_info);
|
|
set_stmt_info (ann, NULL);
|
|
bsi_remove (&si);
|
|
continue;
|
|
}
|
|
|
|
bsi_next (&si);
|
|
} /* stmts in BB */
|
|
} /* BBs in loop */
|
|
|
|
slpeel_make_loop_iterate_ntimes (loop, ratio);
|
|
|
|
/* The memory tags and pointers in vectorized statements need to
|
|
have their SSA forms updated. FIXME, why can't this be delayed
|
|
until all the loops have been transformed? */
|
|
update_ssa (TODO_update_ssa);
|
|
|
|
if (vect_print_dump_info (REPORT_VECTORIZED_LOOPS, LOOP_LOC (loop_vinfo)))
|
|
fprintf (vect_dump, "LOOP VECTORIZED.");
|
|
}
|