diff --git a/gas/ChangeLog b/gas/ChangeLog
index 83f0e6f428..835a5525d6 100644
--- a/gas/ChangeLog
+++ b/gas/ChangeLog
@@ -1,3 +1,16 @@
+2020-06-25  Jan Beulich  <jbeulich@suse.com>
+
+	* config/tc-i386.c (cpu_flags_match): Only match SSE2AVX
+	templates when there's no data size prefix.
+	(md_assemble): Reject data size prefix also for legacy encoded
+	SIMD templates.
+	* testsuite/gas/i386/prefix32.s, testsuite/gas/i386/prefix64.s:
+	Uncomment previously not working line.
+	* testsuite/gas/i386/sse2avx.s: Add ldmxcsr/stmxcsr cases with
+	data16 prefix.
+	* testsuite/gas/i386/prefix32.l, testsuite/gas/i386/prefix64.l,
+	testsuite/gas/i386/sse2avx.d: Adjust expectations.
+
 2020-06-25  Jan Beulich  <jbeulich@suse.com>
 
 	* config/tc-i386.c (build_evex_prefix): Drop early setting of
diff --git a/gas/config/tc-i386.c b/gas/config/tc-i386.c
index 18f4a243d1..5a652a9b9b 100644
--- a/gas/config/tc-i386.c
+++ b/gas/config/tc-i386.c
@@ -1944,7 +1944,8 @@ cpu_flags_match (const insn_template *t)
 	    {
 	      /* We need to check a few extra flags with AVX.  */
 	      if (cpu.bitfield.cpuavx
-		  && (!t->opcode_modifier.sse2avx || sse2avx)
+		  && (!t->opcode_modifier.sse2avx
+		      || (sse2avx && !i.prefix[DATA_PREFIX]))
 		  && (!x.bitfield.cpuaes || cpu.bitfield.cpuaes)
 		  && (!x.bitfield.cpugfni || cpu.bitfield.cpugfni)
 		  && (!x.bitfield.cpupclmul || cpu.bitfield.cpupclmul))
@@ -4787,8 +4788,11 @@ md_assemble (char *line)
       return;
     }
 
-  /* Check for data size prefix on VEX/XOP/EVEX encoded insns.  */
-  if (i.prefix[DATA_PREFIX] && is_any_vex_encoding (&i.tm))
+  /* Check for data size prefix on VEX/XOP/EVEX encoded and SIMD insns.  */
+  if (i.prefix[DATA_PREFIX]
+      && (is_any_vex_encoding (&i.tm)
+	  || i.tm.operand_types[i.imm_operands].bitfield.class >= RegMMX
+	  || i.tm.operand_types[i.imm_operands + 1].bitfield.class >= RegMMX))
     {
       as_bad (_("data size prefix invalid with `%s'"), i.tm.name);
       return;
diff --git a/gas/testsuite/gas/i386/prefix32.l b/gas/testsuite/gas/i386/prefix32.l
index 09a00cd03d..e43abbdc29 100644
--- a/gas/testsuite/gas/i386/prefix32.l
+++ b/gas/testsuite/gas/i386/prefix32.l
@@ -5,7 +5,8 @@
 .*:9: Error: invalid .* `vaddss' after `repne'
 .*:14: Error: same type of prefix .*
 .*:15: Error: same type of prefix .*
-.*:19: Error: same type of prefix .*
+.*:18: Error: data size .* `addps'
+.*:19: Error: data size .* `addpd'
 .*:20: Error: data size .* `vaddps'
 .*:21: Error: data size .* `vaddpd'
 .*:25: Error: same type of prefix .*
@@ -28,9 +29,8 @@ GAS LISTING .*
 [ 	]*15[ 	]+rep bnd ret
 [ 	]*16[ 	]*
 [ 	]*17[ 	]+\.Ldata16:
-[ 	]*18[ 	]+#bogus	data16 addps	%xmm0, %xmm0
-[ 	]*19[ 	]+\?\?\?\? 660F58C0[ 	]+data16 addpd	%xmm0, %xmm0
-\*\*\*\*  Error: .*
+[ 	]*18[ 	]+data16 addps	%xmm0, %xmm0
+[ 	]*19[ 	]+data16 addpd	%xmm0, %xmm0
 [ 	]*20[ 	]+data16 vaddps	%xmm0, %xmm0, %xmm0
 [ 	]*21[ 	]+data16 vaddpd	%xmm0, %xmm0, %xmm0
 [ 	]*22[ 	]*
diff --git a/gas/testsuite/gas/i386/prefix32.s b/gas/testsuite/gas/i386/prefix32.s
index a4382a2b61..598b0a775b 100644
--- a/gas/testsuite/gas/i386/prefix32.s
+++ b/gas/testsuite/gas/i386/prefix32.s
@@ -15,7 +15,7 @@ prefix:
 	rep bnd ret
 
 .Ldata16:
-#bogus	data16 addps	%xmm0, %xmm0
+	data16 addps	%xmm0, %xmm0
 	data16 addpd	%xmm0, %xmm0
 	data16 vaddps	%xmm0, %xmm0, %xmm0
 	data16 vaddpd	%xmm0, %xmm0, %xmm0
diff --git a/gas/testsuite/gas/i386/prefix64.l b/gas/testsuite/gas/i386/prefix64.l
index 0659d9c04d..712f4e0d68 100644
--- a/gas/testsuite/gas/i386/prefix64.l
+++ b/gas/testsuite/gas/i386/prefix64.l
@@ -5,7 +5,8 @@
 .*:9: Error: invalid .* `vaddss' after `repne'
 .*:14: Error: same type of prefix .*
 .*:15: Error: same type of prefix .*
-.*:19: Error: same type of prefix .*
+.*:18: Error: data size .* `addps'
+.*:19: Error: data size .* `addpd'
 .*:20: Error: data size .* `vaddps'
 .*:21: Error: data size .* `vaddpd'
 GAS LISTING .*
@@ -27,10 +28,9 @@ GAS LISTING .*
 [ 	]*15[ 	]+rep bnd ret
 [ 	]*16[ 	]*
 [ 	]*17[ 	]+\.Ldata16:
-[ 	]*18[ 	]+#bogus	data16 addps	%xmm0, %xmm0
-[ 	]*19[ 	]+\?\?\?\? 660F58C0[ 	]+data16 addpd	%xmm0, %xmm0
-\*\*\*\*  Error: .*
+[ 	]*18[ 	]+data16 addps	%xmm0, %xmm0
+[ 	]*19[ 	]+data16 addpd	%xmm0, %xmm0
 [ 	]*20[ 	]+data16 vaddps	%xmm0, %xmm0, %xmm0
 [ 	]*21[ 	]+data16 vaddpd	%xmm0, %xmm0, %xmm0
 [ 	]*22[ 	]*
-[ 	]*23[ 	]*[\?]+ 0+[ \t]+\.p2align	4,0
+#pass
diff --git a/gas/testsuite/gas/i386/prefix64.s b/gas/testsuite/gas/i386/prefix64.s
index 39fb477435..32091c7c05 100644
--- a/gas/testsuite/gas/i386/prefix64.s
+++ b/gas/testsuite/gas/i386/prefix64.s
@@ -15,7 +15,7 @@ prefix:
 	rep bnd ret
 
 .Ldata16:
-#bogus	data16 addps	%xmm0, %xmm0
+	data16 addps	%xmm0, %xmm0
 	data16 addpd	%xmm0, %xmm0
 	data16 vaddps	%xmm0, %xmm0, %xmm0
 	data16 vaddpd	%xmm0, %xmm0, %xmm0
diff --git a/gas/testsuite/gas/i386/sse2avx.d b/gas/testsuite/gas/i386/sse2avx.d
index d03549125b..1a7df7165b 100644
--- a/gas/testsuite/gas/i386/sse2avx.d
+++ b/gas/testsuite/gas/i386/sse2avx.d
@@ -9,6 +9,8 @@ Disassembly of section .text:
 0+ <_start>:
 [ 	]*[a-f0-9]+:	c5 f8 ae 11          	vldmxcsr \(%ecx\)
 [ 	]*[a-f0-9]+:	c5 f8 ae 19          	vstmxcsr \(%ecx\)
+[ 	]*[a-f0-9]+:	66 0f ae 11          	data16 ldmxcsr \(%ecx\)
+[ 	]*[a-f0-9]+:	66 0f ae 19          	data16 stmxcsr \(%ecx\)
 [ 	]*[a-f0-9]+:	c5 f8 5b f4          	vcvtdq2ps %xmm4,%xmm6
 [ 	]*[a-f0-9]+:	c5 f8 5b 21          	vcvtdq2ps \(%ecx\),%xmm4
 [ 	]*[a-f0-9]+:	c5 fb e6 f4          	vcvtpd2dq %xmm4,%xmm6
diff --git a/gas/testsuite/gas/i386/sse2avx.s b/gas/testsuite/gas/i386/sse2avx.s
index 00080cf55b..0ad600c328 100644
--- a/gas/testsuite/gas/i386/sse2avx.s
+++ b/gas/testsuite/gas/i386/sse2avx.s
@@ -7,6 +7,10 @@ _start:
 	ldmxcsr (%ecx)
 	stmxcsr (%ecx)
 
+# These should not be converted
+	data16 ldmxcsr (%ecx)
+	data16 stmxcsr (%ecx)
+
 # Tests for op xmm/mem128, xmm
 	cvtdq2ps %xmm4,%xmm6
 	cvtdq2ps (%ecx),%xmm4