x86: correct operand size match checks for BMI/BMI2 insns

Some BMI/BMI2 insns allow their middle operands to be a memory one. In
such a case, matching register types between operands 0 and 1 as well as
1 and 2 won't help - operands 0 and 2 also need to be checked.

gas/ChangeLog

@@ -1,3 +1,11 @@
+2018-03-08  Jan Beulich  <jbeulich@suse.com>
+
+	* config/tc-i386.c (match_template): Also match register
+	operands 0 and 2 for 3-operand forms.
+	* testsuite/gas/i386/unspec64.l, testsuite/gas/i386/unspec64.s:
+	New.
+	* testsuite/gas/i386/i386.exp: Run new test.
+
 2018-03-08  Jan Beulich  <jbeulich@suse.com>
 
 	* config/tc-i386.c (process_suffix): Do common part of register

gas/config/tc-i386.c

@@ -5492,15 +5492,17 @@ check_reverse:
 		  /* Fall through.  */
 		case 3:
 		  /* Here we make use of the fact that there are no
-		     reverse match 3 operand instructions, and all 3
-		     operand instructions only need to be checked for
-		     register consistency between operands 2 and 3.  */
+		     reverse match 3 operand instructions.  */
 		  if (!operand_type_match (overlap2, i.types[2])
 		      || (check_register
-			  && !operand_type_register_match (i.types[1],
-							   operand_types[1],
-							   i.types[2],
-							   operand_types[2])))
+			  && (!operand_type_register_match (i.types[0],
+							    operand_types[0],
+							    i.types[2],
+							    operand_types[2])
+			      || !operand_type_register_match (i.types[1],
+							       operand_types[1],
+							       i.types[2],
+							       operand_types[2]))))
 		    continue;
 		  break;
 		}

gas/testsuite/gas/i386/i386.exp

@@ -662,6 +662,7 @@ if [expr ([istarget "i*86-*-*"] || [istarget "x86_64-*-*"]) && [gas_64_check]] t
     run_list_test "x86-64-specific-reg"
     run_list_test "suffix-bad"
     run_list_test "x86-64-suffix-bad"
+    run_list_test "unspec64" ""
     run_dump_test "x86-64-fxsave"
     run_dump_test "x86-64-fxsave-intel"
     run_dump_test "x86-64-arch-1"

gas/testsuite/gas/i386/unspec64.l

@@ -0,0 +1,21 @@
+.*: Assembler messages:
+.*:3: Error: .*bextr.*
+.*:4: Error: .*bextr.*
+.*:5: Error: .*bzhi.*
+.*:6: Error: .*bzhi.*
+.*:7: Error: .*sarx.*
+.*:8: Error: .*sarx.*
+.*:9: Error: .*shlx.*
+.*:10: Error: .*shlx.*
+.*:11: Error: .*shrx.*
+.*:12: Error: .*shrx.*
+.*:16: Error: .*bextr.*
+.*:17: Error: .*bextr.*
+.*:18: Error: .*bzhi.*
+.*:19: Error: .*bzhi.*
+.*:20: Error: .*sarx.*
+.*:21: Error: .*sarx.*
+.*:22: Error: .*shlx.*
+.*:23: Error: .*shlx.*
+.*:24: Error: .*shrx.*
+.*:25: Error: .*shrx.*

gas/testsuite/gas/i386/unspec64.s

@@ -0,0 +1,25 @@
+	.text
+unspec:
+	bextr	%eax, (%rax), %rax
+	bextr	%rax, (%rax), %eax
+	bzhi	%eax, (%rax), %rax
+	bzhi	%rax, (%rax), %eax
+	sarx	%eax, (%rax), %rax
+	sarx	%rax, (%rax), %eax
+	shlx	%eax, (%rax), %rax
+	shlx	%rax, (%rax), %eax
+	shrx	%eax, (%rax), %rax
+	shrx	%rax, (%rax), %eax
+
+	.intel_syntax noprefix
+
+	bextr	eax, [rax], rax
+	bextr	rax, [rax], eax
+	bzhi	eax, [rax], rax
+	bzhi	rax, [rax], eax
+	sarx	eax, [rax], rax
+	sarx	rax, [rax], eax
+	shlx	eax, [rax], rax
+	shlx	rax, [rax], eax
+	shrx	eax, [rax], rax
+	shrx	rax, [rax], eax