diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index d0ebc8e53ed..0255b93f26d 100644
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,3 +1,9 @@
+2016-09-27  Oleg Endo  <olegendo@gcc.gnu.org>
+
+	PR target/51244
+	* config/sh/sh.c (sh_rtx_costs): Fix return value of SET of movt and
+	movrt patterns.  Match them before anything else in the SET case.
+
 2016-09-27  Martin Liska  <mliska@suse.cz>
 
 	PR gcov-profile/7970
diff --git a/gcc/config/sh/sh.c b/gcc/config/sh/sh.c
index bfa248d56df..e5b8088c3ef 100644
--- a/gcc/config/sh/sh.c
+++ b/gcc/config/sh/sh.c
@@ -3199,6 +3199,12 @@ sh_rtx_costs (rtx x, machine_mode mode ATTRIBUTE_UNUSED, int outer_code,
 	 vector-move, so we have to provide the correct cost in the number
 	 of move insns to load/store the reg of the mode in question.  */
     case SET:
+      if (sh_movt_set_dest (x) != NULL || sh_movrt_set_dest (x) != NULL)
+	{
+	  *total = COSTS_N_INSNS (1);
+	  return true;
+	}
+
       if (register_operand (SET_DEST (x), VOIDmode)
 	    && (register_operand (SET_SRC (x), VOIDmode)
 		|| satisfies_constraint_Z (SET_SRC (x))))
@@ -3208,10 +3214,6 @@ sh_rtx_costs (rtx x, machine_mode mode ATTRIBUTE_UNUSED, int outer_code,
 				  / mov_insn_size (mode, TARGET_SH2A));
 	  return true;
         }
-
-      if (sh_movt_set_dest (x) != NULL || sh_movrt_set_dest (x) != NULL)
-	return COSTS_N_INSNS (1);
-
       return false;
 
     /* The cost of a mem access is mainly the cost of the address mode.  */