inhibit enum layout optimizations under #[repr(C)] or #[repr(u8)]

Fixes #40029

src/librustc/ty/layout.rs

@@ -1201,7 +1201,8 @@ impl<'a, 'gcx, 'tcx> Layout {
                     });
                 }
 
-                if !def.is_enum() || def.variants.len() == 1 {
+                if !def.is_enum() || (def.variants.len() == 1 &&
+                                      !def.repr.inhibit_enum_layout_opt()) {
                     // Struct, or union, or univariant enum equivalent to a struct.
                     // (Typechecking will reject discriminant-sizing attrs.)
 
@@ -1250,7 +1251,7 @@ impl<'a, 'gcx, 'tcx> Layout {
                     v.fields.iter().map(|field| field.ty(tcx, substs)).collect::<Vec<_>>()
                 }).collect::<Vec<_>>();
 
-                if variants.len() == 2 && !def.repr.c {
+                if variants.len() == 2 && !def.repr.inhibit_enum_layout_opt() {
                     // Nullable pointer optimization
                     for discr in 0..2 {
                         let other_fields = variants[1 - discr].iter().map(|ty| {

src/librustc/ty/mod.rs

@@ -1391,6 +1391,13 @@ impl ReprOptions {
     pub fn discr_type(&self) -> attr::IntType {
         self.int.unwrap_or(attr::SignedInt(ast::IntTy::Is))
     }
+
+    /// Returns true if this `#[repr()]` should inhabit "smart enum
+    /// layout" optimizations, such as representing `Foo<&T>` as a
+    /// single pointer.
+    pub fn inhibit_enum_layout_opt(&self) -> bool {
+        self.c || self.int.is_some()
+    }
 }
 
 impl<'a, 'gcx, 'tcx> AdtDef {

src/test/run-pass/enum-layout-optimization.rs

@@ -0,0 +1,59 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// Test that we will do various size optimizations to enum layout, but
+// *not* if `#[repr(u8)]` or `#[repr(C)]` is passed. See also #40029.
+
+#![allow(dead_code)]
+
+use std::mem;
+
+enum Nullable<T> {
+    Alive(T),
+    Dropped,
+}
+
+#[repr(u8)]
+enum NullableU8<T> {
+    Alive(T),
+    Dropped,
+}
+
+#[repr(C)]
+enum NullableC<T> {
+    Alive(T),
+    Dropped,
+}
+
+struct StructNewtype<T>(T);
+
+#[repr(C)]
+struct StructNewtypeC<T>(T);
+
+enum EnumNewtype<T> { Variant(T) }
+
+#[repr(u8)]
+enum EnumNewtypeU8<T> { Variant(T) }
+
+#[repr(C)]
+enum EnumNewtypeC<T> { Variant(T) }
+
+fn main() {
+    assert!(mem::size_of::<Box<i32>>() == mem::size_of::<Nullable<Box<i32>>>());
+    assert!(mem::size_of::<Box<i32>>() < mem::size_of::<NullableU8<Box<i32>>>());
+    assert!(mem::size_of::<Box<i32>>() < mem::size_of::<NullableC<Box<i32>>>());
+
+    assert!(mem::size_of::<i32>() == mem::size_of::<StructNewtype<i32>>());
+    assert!(mem::size_of::<i32>() == mem::size_of::<StructNewtypeC<i32>>());
+
+    assert!(mem::size_of::<i32>() == mem::size_of::<EnumNewtype<i32>>());
+    assert!(mem::size_of::<i32>() < mem::size_of::<EnumNewtypeU8<i32>>());
+    assert!(mem::size_of::<i32>() < mem::size_of::<EnumNewtypeC<i32>>());
+}